diff --git a/.circleci/circle_requirements.txt b/.circleci/circle_requirements.txt deleted file mode 100644 index 1c010d2..0000000 --- a/.circleci/circle_requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -poetry>=1.1.6 -tox>=3.23.1 -tox-poetry>=0.3.0 diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 702b12d..0000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,139 +0,0 @@ -version: 2.1 - -commands: - - abort_for_docs: - steps: - - run: - name: Avoid tests for docs - command: | - if [[ $CIRCLE_BRANCH == *docs ]]; then - echo "Identifies as documents PR, no testing required" - circleci step halt - fi - - abort_for_noci: - steps: - - run: - name: Ignore CI for specific branches - command: | - if [[ $CIRCLE_BRANCH == *noci ]]; then - echo "Identifies as actively ignoring CI, no testing required." - circleci step halt - fi - - - early_return_for_forked_pull_requests: - description: >- - If this build is from a fork, stop executing the current job and return success. - This is useful to avoid steps that will fail due to missing credentials. - steps: - - run: - name: Early return if this build is from a forked PR - command: | - if [[ -n "$CIRCLE_PR_NUMBER" ]]; then - echo "Nothing to do for forked PRs, so marking this step successful" - circleci step halt - fi - - build_and_test: - steps: - - checkout - - restore_cache: # Download and cache dependencies - keys: - - v1-dependencies-{{ checksum "pyproject.toml" }} - # fallback to using the latest cache if no exact match is found - - v1-dependencies- - - - run: - name: install tox dependencies - command: | - pip install --user --quiet -r .circleci/circle_requirements.txt - - - run: - name: build sdist and wheels - command: | - poetry build - - - run: - name: lint - command: | - tox -e linters - - - run: - name: run tests - command: - tox -e tests - - - save_cache: - paths: - - ./.tox - - ~/.cache/pip - key: v1-dependencies-{{ checksum "pyproject.toml" }} - -jobs: - build: - parameters: - python_version: - type: string - default: "latest" - docker: - - image: circleci/python:<> - - image: redislabs/redisai:edge-cpu-bionic - - steps: - - build_and_test - - store_artifacts: - path: test-reports - destination: test-reports - - nightly: - parameters: - python_version: - type: string - docker: - - image: circleci/python:<> - - image: redislabs/redisai:edge-cpu-bionic - steps: - - build_and_test - - dockerize - -on-any-branch: &on-any-branch - filters: - branches: - only: - - /.*/ - tags: - ignore: /.*/ - -on-master: &on-master - filters: - branches: - only: - - master - -python-versions: &python-versions - matrix: - parameters: - python_version: - - "3.6.9" - - "3.7.9" - - "3.8.9" - - "3.9.4" - - "latest" - -workflows: - version: 2 - commit: - jobs: - - build: - <<: *on-any-branch - <<: *python-versions - - nightly: - triggers: - - schedule: - cron: "0 0 * * *" - <<: *on-master - jobs: - - build diff --git a/.github/release-drafter-config.yml b/.github/release-drafter-config.yml index 84791cc..0eb1ee3 100644 --- a/.github/release-drafter-config.yml +++ b/.github/release-drafter-config.yml @@ -1,7 +1,7 @@ name-template: 'Version $NEXT_PATCH_VERSION' tag-template: 'v$NEXT_PATCH_VERSION' categories: - - title: '🚀Features' + - title: 'Features' labels: - 'feature' - 'enhancement' @@ -12,7 +12,7 @@ categories: - 'bug' - title: 'Maintenance' label: 'chore' -change-template: '- $TITLE @$AUTHOR (#$NUMBER)' +change-template: '- $TITLE (#$NUMBER)' exclude-labels: - 'skip-changelog' template: | diff --git a/.github/workflows/check-pypi.yml b/.github/workflows/check-pypi.yml index e1d5b10..074f15c 100644 --- a/.github/workflows/check-pypi.yml +++ b/.github/workflows/check-pypi.yml @@ -1,6 +1,11 @@ name: Check if required secrets are set to publish to Pypi -on: push +on: + push: + branches: + - 'master' + - 'main' + - '[0-9].[0-9]' jobs: checksecret: diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml new file mode 100644 index 0000000..86223bd --- /dev/null +++ b/.github/workflows/integration.yml @@ -0,0 +1,46 @@ +--- + +on: + push: + + schedule: + - cron: "5 1 * * Sun-Thu" + +name: tests + +jobs: + build-and-test: + + services: + redisai: + image: redislabs/redisai:edge-cpu-bionic + ports: + - 6379:6379 + runs-on: ubuntu-latest + + strategy: + max-parallel: 10 + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11"] + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: ${{matrix.python-version}} + + # penidng: https://github.com/tkukushkin/tox-poetry/pull/16 + - name: install base dependencies + run: | + pip install -q tox==3.27.0 poetry tox-poetry + - name: cache + uses: actions/cache@v3 + with: + path: | + .tox + key: redisai-${{matrix.python_version}} + - name: build the package + run: | + poetry build + - name: test + run: | + tox -e tests diff --git a/.github/workflows/linters.yml b/.github/workflows/linters.yml new file mode 100644 index 0000000..fd9d9e4 --- /dev/null +++ b/.github/workflows/linters.yml @@ -0,0 +1,25 @@ +--- + +on: + pull_request: + paths: + - 'redisai/**' + - 'pyproject.toml' + +name: lint + +env: + python_version: 3.9 + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: ${{env.python_version}} + - name: lint + run: | + pip install -q tox poetry + tox -e linters diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index 7caeccd..8b4f937 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -11,13 +11,29 @@ jobs: ACTIONS_ALLOW_UNSECURE_COMMANDS: true steps: - uses: actions/checkout@master - - name: Set up Python 3.7 + + - name: get version from tag + id: get_version + run: | + realversion="${GITHUB_REF/refs\/tags\//}" + realversion="${realversion//v/}" + echo "::set-output name=VERSION::$realversion" + + - name: Set the version for publishing + uses: ciiiii/toml-editor@1.0.0 + with: + file: "pyproject.toml" + key: "tool.poetry.version" + value: "${{ steps.get_version.outputs.VERSION }}" + + - name: Set up Python 3.8 uses: actions/setup-python@v1 with: - python-version: 3.7 + python-version: 3.8 - - name: Install Poetry - uses: dschep/install-poetry-action@v1.3 + - name: install poetry + run: | + pip install poetry - name: Cache Poetry virtualenv uses: actions/cache@v1 diff --git a/.gitignore b/.gitignore index 343d72a..f3fdf66 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ .pydevproject *.pyc .venv/ +venv/ redisai.egg-info .idea .mypy_cache/ @@ -9,3 +10,4 @@ build/ dist/ docs/_build/ .DS_Store +.vscode diff --git a/README.rst b/README.rst index 3d05052..8b12efa 100644 --- a/README.rst +++ b/README.rst @@ -8,8 +8,8 @@ redisai-py .. image:: https://badge.fury.io/py/redisai.svg :target: https://badge.fury.io/py/redisai -.. image:: https://circleci.com/gh/RedisAI/redisai-py/tree/master.svg?style=svg - :target: https://circleci.com/gh/RedisAI/redisai-py/tree/master +.. image:: https://github.com/RedisAI/redisai-py/actions/workflows/integration.yml/badge.svg + :target: https://github.com/RedisAI/redisai-py/actions/workflows/integration.yml .. image:: https://img.shields.io/github/release/RedisAI/redisai-py.svg :target: https://github.com/RedisAI/redisai-py/releases/latest @@ -21,12 +21,13 @@ redisai-py :target: https://redisai-py.readthedocs.io/en/latest/?badge=latest .. image:: https://img.shields.io/badge/Forum-RedisAI-blue - :target: https://forum.redislabs.com/c/modules/redisai + :target: https://forum.redis.com/c/modules/redisai .. image:: https://img.shields.io/discord/697882427875393627?style=flat-square :target: https://discord.gg/rTQm7UZ -.. image:: https://snyk.io/test/github/RedisAI/redisai-py/badge.svg?targetFile=pyproject.toml)](https://snyk.io/test/github/RedisAI/redisai-py?targetFile=pyproject.toml +.. image:: https://snyk.io/test/github/RedisAI/redisai-py/badge.svg?targetFile=pyproject.toml + :target: https://snyk.io/test/github/RedisAI/redisai-py?targetFile=pyproject.toml redisai-py is the Python client for RedisAI. Checkout the `documentation `_ for API details and examples diff --git a/docs/conf.py b/docs/conf.py index 9897f38..0d5acc9 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,6 +1,6 @@ project = "redisai-py" -copyright = "2020, RedisLabs" -author = "RedisLabs" +copyright = "2020, Redis" +author = "Redis" release = "1.0.2" extensions = [ "sphinx.ext.autodoc", @@ -10,15 +10,21 @@ "sphinx.ext.todo", "sphinx.ext.intersphinx", "sphinx_rtd_theme", + 'sphinx_search.extension', # search tools + 'sphinx.ext.autodoc', ] +pygments_style = "sphinx" +autoapi_type = 'python' templates_path = ["_templates"] source_suffix = ".rst" master_doc = "index" exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] html_theme = "sphinx_rtd_theme" +html_logo = 'images/logo.png' html_use_smartypants = True html_last_updated_fmt = "%b %d, %Y" html_split_index = False +html_static_path = ['_static'] html_sidebars = { "**": ["searchbox.html", "globaltoc.html", "sourcelink.html"], } diff --git a/docs/images/logo.png b/docs/images/logo.png new file mode 100644 index 0000000..f9ee8bf Binary files /dev/null and b/docs/images/logo.png differ diff --git a/pyproject.toml b/pyproject.toml index e2cfe99..143834e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,8 +1,8 @@ [tool.poetry] name = "redisai" -version = "1.0.2" +version = "1.3.0" description = "RedisAI Python Client" -authors = ["RedisLabs "] +authors = ["Redis "] license = "BSD-3-Clause" readme = "README.rst" @@ -14,38 +14,45 @@ classifiers = [ 'Topic :: Database', 'Programming Language :: Python', 'Intended Audience :: Developers', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', 'License :: OSI Approved :: BSD License', 'Development Status :: 5 - Production/Stable' ] [tool.poetry.dependencies] -python = "^3.6" -redis = ">=2.10" +python = ">=3.8,<=4.0.0" +redis = "^4.1.4" hiredis = ">=0.20" numpy = ">=1.19.5" six = ">=1.10.0" Deprecated = "^1.2.12" +pytest = "^7.2.1" [tool.poetry.dev-dependencies] codecov = "^2.1.11" -flake8 = "^3.9.2" -rmtest = "^0.7.0" -nose = "^1.3.7" +flake8 = "<6.0.0" ml2rt = "^0.2.0" -tox = ">=3.23.1" +tox = ">=3.23.1,<=4.0.0" tox-poetry = "^0.3.0" +Sphinx = "^4.1.2" +sphinx-rtd-theme = "^0.5.2" +readthedocs-sphinx-search = "^0.1.0" +sphinx-autoapi = "^1.8.3" +toml = "^0.10.2" bandit = "^1.7.0" pylint = "^2.8.2" vulture = "^2.3" +scikit-image = "^0.19.3" [tool.poetry.urls] -url = "https://redisai.io" -repository = "https://github.com/RedisAI/redisai-py" - +"Project URL" = "https://redisai.io" +Repository = "https://github.com/RedisAI/redisai-py" +Documentation = "https://redisai.readhtedocs.io" +Homepage = "https://oss..com/redisai/" +Tracker = "https://github.com/RedisAI/redisai-py/issues" [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/redisai/__init__.py b/redisai/__init__.py index a03dbac..a72de04 100644 --- a/redisai/__init__.py +++ b/redisai/__init__.py @@ -1,3 +1,4 @@ from .client import Client # noqa +import pkg_resources -__version__ = "1.0.2" +__version__ = pkg_resources.get_distribution('redisai').version diff --git a/redisai/client.py b/redisai/client.py index 1411ad8..8730d36 100644 --- a/redisai/client.py +++ b/redisai/client.py @@ -70,7 +70,12 @@ def pipeline(self, transaction: bool = True, shard_hint: bool = None) -> "Pipeli ) def dag( - self, load: Sequence = None, persist: Sequence = None, readonly: bool = False + self, + load: Sequence = None, + persist: Sequence = None, + routing: AnyStr = None, + timeout: int = None, + readonly: bool = False ) -> "Dag": """ It returns a DAG object on which other DAG-allowed operations can be called. For @@ -81,7 +86,16 @@ def dag( load : Union[AnyStr, List[AnyStr]] Load the list of given values from the keyspace to DAG scope persist : Union[AnyStr, List[AnyStr]] - Write the list of given key, values to the keyspace from DAG scope + For each tensor key in the given list, write its values to the keyspace from + DAG scope after the DAG execution is finished. + routing : AnyStr + Denotes a key to be used in the DAG or a tag that will assist in routing the dag + execution command to the right shard. Redis will verify that all potential key + accesses are done to within the target shard. + timeout : int + The max number on milisecinds that may pass before the request is prossced + (meaning that the result will not be computed after that time and TIMEDOUT + is returned in that case) readonly : bool If True, it triggers AI.DAGRUN_RO, the read only DAG which cannot write (PERSIST) to the keyspace. But since it can't write, it can execute on replicas @@ -105,9 +119,7 @@ def dag( >>> # You can even chain the operations >>> result = dag.tensorset(**akwargs).modelrun(**bkwargs).tensorget(**ckwargs).run() """ - return Dag( - load, persist, self.execute_command, readonly, self.enable_postprocess - ) + return Dag(load, persist, routing, timeout, self.execute_command, readonly) def loadbackend(self, identifier: AnyStr, path: AnyStr) -> str: """ @@ -133,9 +145,44 @@ def loadbackend(self, identifier: AnyStr, path: AnyStr) -> str: 'OK' """ args = builder.loadbackend(identifier, path) - res = self.execute_command(*args) + res = self.execute_command(args) return res if not self.enable_postprocess else processor.loadbackend(res) + def config(self, name: str, value: Union[str, int, None] = None) -> str: + """ + Get/Set configuration item. Current available configurations are: BACKENDSPATH and MODEL_CHUNK_SIZE. + For more details, see: https://oss.redis.com/redisai/master/commands/#aiconfig. + If value is given - the configuration under name will be overriten. + + Parameters + ---------- + name: str + RedisAI config item to retreive/override (BACKENDSPATH / MODEL_CHUNK_SIZE). + value: Union[str, int] + Value to set the config item with (if given). + + Returns + ------- + The current configuration value if value is None, + 'OK' if value was given and configuration overitten succeeded, + raise an exception otherwise + + + Example + ------- + >>> con.config('MODEL_CHUNK_SIZE', 128 * 1024) + 'OK' + >>> con.config('BACKENDSPATH', '/my/backends/path') + 'OK' + >>> con.config('BACKENDSPATH') + '/my/backends/path' + >>> con.config('MODEL_CHUNK_SIZE') + '131072' + """ + args = builder.config(name, value) + res = self.execute_command(args) + return res if not self.enable_postprocess or not isinstance(res, bytes) else processor.config(res) + def modelstore( self, key: AnyStr, @@ -197,6 +244,7 @@ def modelstore( ... inputs=['a', 'b'], outputs=['mul'], tag='v1.0') 'OK' """ + chunk_size = self.config('MODEL_CHUNK_SIZE') args = builder.modelstore( key, backend, @@ -208,6 +256,7 @@ def modelstore( tag, inputs, outputs, + chunk_size=chunk_size ) res = self.execute_command(*args) return res if not self.enable_postprocess else processor.modelstore(res) diff --git a/redisai/command_builder.py b/redisai/command_builder.py index 200a4d3..bb65882 100644 --- a/redisai/command_builder.py +++ b/redisai/command_builder.py @@ -8,7 +8,13 @@ def loadbackend(identifier: AnyStr, path: AnyStr) -> Sequence: - return "AI.CONFIG LOADBACKEND", identifier, path + return f'AI.CONFIG LOADBACKEND {identifier} {path}' + + +def config(name: str, value: Union[str, int, None] = None) -> Sequence: + if value is not None: + return f'AI.CONFIG {name} {value}' + return f'AI.CONFIG GET {name}' def modelstore( @@ -22,10 +28,14 @@ def modelstore( tag: AnyStr, inputs: Union[AnyStr, List[AnyStr]], outputs: Union[AnyStr, List[AnyStr]], + chunk_size: int = 500 * 1024 * 1024 ) -> Sequence: if name is None: raise ValueError("Model name was not given") - if device.upper() not in utils.allowed_devices: + + # device format should be: "CPU | GPU [:]" + device_type = device.split(":")[0] + if device_type.upper() not in utils.allowed_devices: raise ValueError(f"Device not allowed. Use any from {utils.allowed_devices}") if backend.upper() not in utils.allowed_backends: raise ValueError(f"Backend not allowed. Use any from {utils.allowed_backends}") @@ -63,9 +73,7 @@ def modelstore( raise ValueError( "Inputs and outputs keywords should not be specified for this backend" ) - chunk_size = 500 * 1024 * 1024 # TODO: this should be configurable. data_chunks = [data[i: i + chunk_size] for i in range(0, len(data), chunk_size)] - # TODO: need a test case for this args += ["BLOB", *data_chunks] return args @@ -173,7 +181,12 @@ def tensorset( args = ["AI.TENSORSET", key, dtype, *shape, "BLOB", blob] elif isinstance(tensor, (list, tuple)): try: - dtype = utils.dtype_dict[dtype.lower()] + # Numpy 'str' dtype has many different names regarding maximal length in the tensor and more, + # but the all share the 'num' attribute. This is a way to check if a dtype is a kind of string. + if np.dtype(dtype).num == np.dtype("str").num: + dtype = utils.dtype_dict["str"] + else: + dtype = utils.dtype_dict[dtype.lower()] except KeyError: raise TypeError( f"``{dtype}`` is not supported by RedisAI. Currently " diff --git a/redisai/dag.py b/redisai/dag.py index 1746010..dcbf1d7 100644 --- a/redisai/dag.py +++ b/redisai/dag.py @@ -5,36 +5,57 @@ from redisai import command_builder as builder from redisai.postprocessor import Processor +from deprecated import deprecated +import warnings processor = Processor() class Dag: - def __init__(self, load, persist, executor, readonly=False, postprocess=True): + def __init__(self, load, persist, routing, timeout, executor, readonly=False): self.result_processors = [] - self.enable_postprocess = postprocess - if readonly: - if persist: - raise RuntimeError( - "READONLY requests cannot write (duh!) and should not " - "have PERSISTing values" - ) - self.commands = ["AI.DAGRUN_RO"] + self.enable_postprocess = True + self.deprecatedDagrunMode = load is None and persist is None and routing is None + self.readonly = readonly + self.executor = executor + + if readonly and persist: + raise RuntimeError( + "READONLY requests cannot write (duh!) and should not " + "have PERSISTing values" + ) + + if self.deprecatedDagrunMode: + # Throw warning about using deprecated dagrun + warnings.warn("Creating Dag without any of LOAD, PERSIST and ROUTING arguments" + "is allowed only in deprecated AI.DAGRUN or AI.DAGRUN_RO commands", DeprecationWarning) + # Use dagrun + if readonly: + self.commands = ["AI.DAGRUN_RO"] + else: + self.commands = ["AI.DAGRUN"] else: - self.commands = ["AI.DAGRUN"] - if load: + # Use dagexecute + if readonly: + self.commands = ["AI.DAGEXECUTE_RO"] + else: + self.commands = ["AI.DAGEXECUTE"] + if load is not None: if not isinstance(load, (list, tuple)): self.commands += ["LOAD", 1, load] else: self.commands += ["LOAD", len(load), *load] - if persist: + if persist is not None: if not isinstance(persist, (list, tuple)): - self.commands += ["PERSIST", 1, persist, "|>"] + self.commands += ["PERSIST", 1, persist] else: - self.commands += ["PERSIST", len(persist), *persist, "|>"] - else: - self.commands.append("|>") - self.executor = executor + self.commands += ["PERSIST", len(persist), *persist] + if routing is not None: + self.commands += ["ROUTING", routing] + if timeout is not None: + self.commands += ["TIMEOUT", timeout] + + self.commands.append("|>") def tensorset( self, @@ -69,20 +90,71 @@ def tensorget( ) return self + @deprecated(version="1.2.0", reason="Use modelexecute instead") def modelrun( + self, + key: AnyStr, + inputs: Union[AnyStr, List[AnyStr]], + outputs: Union[AnyStr, List[AnyStr]], + ) -> Any: + if self.deprecatedDagrunMode: + args = builder.modelrun(key, inputs, outputs) + self.commands.extend(args) + self.commands.append("|>") + self.result_processors.append(bytes.decode) + return self + else: + return self.modelexecute(key, inputs, outputs) + + def modelexecute( self, key: AnyStr, inputs: Union[AnyStr, List[AnyStr]], outputs: Union[AnyStr, List[AnyStr]], ) -> Any: - args = builder.modelrun(key, inputs, outputs) + if self.deprecatedDagrunMode: + raise RuntimeError( + "You are using deprecated version of DAG, that does not supports MODELEXECUTE." + "The new version requires giving at least one of LOAD, PERSIST and ROUTING" + "arguments when constructing the Dag" + ) + args = builder.modelexecute(key, inputs, outputs, None) self.commands.extend(args) self.commands.append("|>") self.result_processors.append(bytes.decode) return self + def scriptexecute( + self, + key: AnyStr, + function: str, + keys: Union[AnyStr, Sequence[AnyStr]] = None, + inputs: Union[AnyStr, Sequence[AnyStr]] = None, + args: Union[AnyStr, Sequence[AnyStr]] = None, + outputs: Union[AnyStr, List[AnyStr]] = None, + ) -> Any: + if self.readonly: + raise RuntimeError( + "AI.SCRIPTEXECUTE cannot be used in readonly mode" + ) + if self.deprecatedDagrunMode: + raise RuntimeError( + "You are using deprecated version of DAG, that does not supports SCRIPTEXECUTE." + "The new version requires giving at least one of LOAD, PERSIST and ROUTING" + "arguments when constructing the Dag" + ) + args = builder.scriptexecute(key, function, keys, inputs, args, outputs, None) + self.commands.extend(args) + self.commands.append("|>") + self.result_processors.append(bytes.decode) + return self + + @deprecated(version="1.2.0", reason="Use execute instead") def run(self): - commands = self.commands[:-1] # removing the last "|> + return self.execute() + + def execute(self): + commands = self.commands[:-1] # removing the last "|>" results = self.executor(*commands) if self.enable_postprocess: out = [] diff --git a/redisai/postprocessor.py b/redisai/postprocessor.py index ae93fab..96a4fcf 100644 --- a/redisai/postprocessor.py +++ b/redisai/postprocessor.py @@ -27,24 +27,28 @@ def tensorget(res, as_numpy, as_numpy_mutable, meta_only): rai_result = utils.list2dict(res) if meta_only is True: return rai_result - elif as_numpy_mutable is True: + if as_numpy_mutable is True: return utils.blob2numpy( rai_result["blob"], rai_result["shape"], rai_result["dtype"], mutable=True, ) - elif as_numpy is True: + if as_numpy is True: return utils.blob2numpy( rai_result["blob"], rai_result["shape"], rai_result["dtype"], mutable=False, ) + + if rai_result["dtype"] == "STRING": + def target(b): + return b.decode() else: target = float if rai_result["dtype"] in ("FLOAT", "DOUBLE") else int - utils.recursive_bytetransform(rai_result["values"], target) - return rai_result + utils.recursive_bytetransform(rai_result["values"], target) + return rai_result @staticmethod def scriptget(res): @@ -62,19 +66,20 @@ def infoget(res): # These functions are only doing decoding on the output from redis decoder = staticmethod(decoder) decoding_functions = ( + "config", + "inforeset", "loadbackend", - "modelstore", - "modelset", "modeldel", "modelexecute", "modelrun", - "tensorset", - "scriptset", - "scriptstore", + "modelset", + "modelstore", "scriptdel", - "scriptrun", "scriptexecute", - "inforeset", + "scriptrun", + "scriptset", + "scriptstore", + "tensorset", ) for fn in decoding_functions: setattr(Processor, fn, decoder) diff --git a/redisai/utils.py b/redisai/utils.py index ba41809..c0720a3 100644 --- a/redisai/utils.py +++ b/redisai/utils.py @@ -14,6 +14,8 @@ "uint16": "UINT16", "uint32": "UINT32", "uint64": "UINT64", + "bool": "BOOL", + "str": "STRING", } allowed_devices = {"CPU", "GPU"} @@ -23,11 +25,15 @@ def numpy2blob(tensor: np.ndarray) -> tuple: """Convert the numpy input from user to `Tensor`.""" try: - dtype = dtype_dict[str(tensor.dtype)] + if tensor.dtype.num == np.dtype("str").num: + dtype = dtype_dict["str"] + blob = "".join([string + "\0" for string in tensor.flat]) + else: + dtype = dtype_dict[str(tensor.dtype)] + blob = tensor.tobytes() except KeyError: raise TypeError(f"RedisAI doesn't support tensors of type {tensor.dtype}") shape = tensor.shape - blob = bytes(tensor.data) return dtype, shape, blob @@ -37,7 +43,9 @@ def blob2numpy( """Convert `BLOB` result from RedisAI to `np.ndarray`.""" mm = {"FLOAT": "float32", "DOUBLE": "float64"} dtype = mm.get(dtype, dtype.lower()) - if mutable: + if dtype == 'string': + a = np.array(value.decode().split('\0')[:-1], dtype='str') + elif mutable: a = np.fromstring(value, dtype=dtype) else: a = np.frombuffer(value, dtype=dtype) diff --git a/test/test.py b/test/test.py index 4690117..e362fab 100644 --- a/test/test.py +++ b/test/test.py @@ -1,7 +1,11 @@ import os.path import sys +import warnings + from io import StringIO from unittest import TestCase +from skimage.io import imread +from skimage.transform import resize import numpy as np from ml2rt import load_model @@ -9,9 +13,11 @@ from redisai import Client + DEBUG = False tf_graph = "graph.pb" torch_graph = "pt-minimal.pt" +dog_img = "dog.jpg" class Capturing(list): @@ -67,19 +73,29 @@ def func(tensors: List[Tensor], keys: List[str], args: List[str]): return b + a """ +data_processing_script = r""" +def pre_process_3ch(tensors: List[Tensor], keys: List[str], args: List[str]): + return tensors[0].float().div(255).unsqueeze(0) + +def post_process(tensors: List[Tensor], keys: List[str], args: List[str]): + # tf model has 1001 classes, hence negative 1 + return tensors[0].max(1)[1] - 1 +""" + class RedisAITestBase(TestCase): def setUp(self): super().setUp() - self.get_client().flushall() + RedisAITestBase.get_client().flushall() - def get_client(self, debug=DEBUG): + @staticmethod + def get_client(debug=DEBUG): return Client(debug) class ClientTestCase(RedisAITestBase): def test_set_non_numpy_tensor(self): - con = self.get_client() + con = RedisAITestBase.get_client() con.tensorset("x", (2, 3, 4, 5), dtype="float") result = con.tensorget("x", as_numpy=False) self.assertEqual([2, 3, 4, 5], result["values"]) @@ -96,6 +112,18 @@ def test_set_non_numpy_tensor(self): self.assertEqual([2, 3, 4, 5], result["values"]) self.assertEqual([2, 2], result["shape"]) + con.tensorset("x", (1, 1, 0, 0), dtype="bool", shape=(2, 2)) + result = con.tensorget("x", as_numpy=False) + self.assertEqual([True, True, False, False], result["values"]) + self.assertEqual([2, 2], result["shape"]) + self.assertEqual("BOOL", result["dtype"]) + + con.tensorset("x", (12, 'a', 'G', 'four'), dtype="str", shape=(2, 2)) + result = con.tensorget("x", as_numpy=False) + self.assertEqual(['12', 'a', 'G', 'four'], result["values"]) + self.assertEqual([2, 2], result["shape"]) + self.assertEqual("STRING", result["dtype"]) + with self.assertRaises(TypeError): con.tensorset("x", (2, 3, 4, 5), dtype="wrongtype", shape=(2, 2)) con.tensorset("x", (2, 3, 4, 5), dtype="int8", shape=(2, 2)) @@ -110,14 +138,14 @@ def test_set_non_numpy_tensor(self): con.tensorset(1) def test_tensorget_meta(self): - con = self.get_client() + con = RedisAITestBase.get_client() con.tensorset("x", (2, 3, 4, 5), dtype="float") result = con.tensorget("x", meta_only=True) self.assertNotIn("values", result) self.assertEqual([4], result["shape"]) def test_numpy_tensor(self): - con = self.get_client() + con = RedisAITestBase.get_client() input_array = np.array([2, 3], dtype=np.float32) con.tensorset("x", input_array) @@ -129,6 +157,18 @@ def test_numpy_tensor(self): values = con.tensorget("x") self.assertEqual(values.dtype, np.float64) + input_array = np.array([True, False]) + con.tensorset("x", input_array) + values = con.tensorget("x") + self.assertEqual(values.dtype, "bool") + self.assertTrue(np.array_equal(values, [True, False])) + + input_array = np.array(["a", "bb", "⚓⚓⚓", "d♻d♻"]).reshape((2, 2)) + con.tensorset("x", input_array) + values = con.tensorget("x") + self.assertEqual(values.dtype.num, np.dtype("str").num) + self.assertTrue(np.array_equal(values, [['a', 'bb'], ["⚓⚓⚓", "d♻d♻"]])) + input_array = np.array([2, 3]) con.tensorset("x", input_array) values = con.tensorget("x") @@ -147,15 +187,11 @@ def test_numpy_tensor(self): np.put(ret, 0, 1) self.assertEqual(ret[0], 1) - stringarr = np.array("dummy") - with self.assertRaises(TypeError): - con.tensorset("trying", stringarr) - # AI.MODELSET is deprecated by AI.MODELSTORE. def test_deprecated_modelset(self): model_path = os.path.join(MODEL_DIR, "graph.pb") model_pb = load_model(model_path) - con = self.get_client() + con = RedisAITestBase.get_client() with self.assertRaises(ValueError): con.modelset( "m", @@ -197,7 +233,7 @@ def test_deprecated_modelset(self): def test_modelstore_errors(self): model_path = os.path.join(MODEL_DIR, "graph.pb") model_pb = load_model(model_path) - con = self.get_client() + con = RedisAITestBase.get_client() with self.assertRaises(ValueError) as e: con.modelstore( @@ -284,7 +320,7 @@ def test_modelstore_errors(self): def test_modelget_meta(self): model_path = os.path.join(MODEL_DIR, tf_graph) model_pb = load_model(model_path) - con = self.get_client() + con = RedisAITestBase.get_client() con.modelstore( "m", "tf", "cpu", model_pb, inputs=["a", "b"], outputs=["mul"], tag="v1.0" ) @@ -306,7 +342,7 @@ def test_modelget_meta(self): def test_modelexecute_non_list_input_output(self): model_path = os.path.join(MODEL_DIR, "graph.pb") model_pb = load_model(model_path) - con = self.get_client() + con = RedisAITestBase.get_client() con.modelstore( "m", "tf", "cpu", model_pb, inputs=["a", "b"], outputs=["mul"], tag="v1.7" ) @@ -315,11 +351,11 @@ def test_modelexecute_non_list_input_output(self): ret = con.modelexecute("m", ["a", "b"], "out") self.assertEqual(ret, "OK") - def test_nonasciichar(self): + def test_non_ascii_char(self): nonascii = "ĉ" model_path = os.path.join(MODEL_DIR, tf_graph) model_pb = load_model(model_path) - con = self.get_client() + con = RedisAITestBase.get_client() con.modelstore( "m" + nonascii, "tf", @@ -336,6 +372,21 @@ def test_nonasciichar(self): tensor = con.tensorget("c" + nonascii) self.assertTrue((np.allclose(tensor, [4.0, 9.0]))) + def test_device_with_id(self): + model_path = os.path.join(MODEL_DIR, tf_graph) + model_pb = load_model(model_path) + con = RedisAITestBase.get_client() + ret = con.modelstore( + "m", + "tf", + "cpu:1", + model_pb, + inputs=["a", "b"], + outputs=["mul"], + tag="v1.0", + ) + self.assertEqual('OK', ret) + def test_run_tf_model(self): model_path = os.path.join(MODEL_DIR, tf_graph) bad_model_path = os.path.join(MODEL_DIR, torch_graph) @@ -343,7 +394,7 @@ def test_run_tf_model(self): model_pb = load_model(model_path) wrong_model_pb = load_model(bad_model_path) - con = self.get_client() + con = RedisAITestBase.get_client() con.modelstore( "m", "tf", "cpu", model_pb, inputs=["a", "b"], outputs=["mul"], tag="v1.0" ) @@ -386,7 +437,7 @@ def test_run_tf_model(self): # AI.SCRIPTRUN is deprecated by AI.SCRIPTEXECUTE # and AI.SCRIPTSET is deprecated by AI.SCRIPTSTORE def test_deprecated_scriptset_and_scriptrun(self): - con = self.get_client() + con = RedisAITestBase.get_client() self.assertRaises(ResponseError, con.scriptset, "scr", "cpu", "return 1") con.scriptset("scr", "cpu", script_old) con.tensorset("a", (2, 3), dtype="float") @@ -403,7 +454,7 @@ def test_deprecated_scriptset_and_scriptrun(self): self.assertEqual([4, 6], tensor["values"]) def test_scriptstore(self): - con = self.get_client() + con = RedisAITestBase.get_client() # try with bad arguments: with self.assertRaises(ValueError) as e: con.scriptstore("test", "cpu", script, entry_points=None) @@ -415,7 +466,7 @@ def test_scriptstore(self): "expected def but found 'return' here: File \"\", line 1 return 1 ~~~~~~ <--- HERE ") def test_scripts_execute(self): - con = self.get_client() + con = RedisAITestBase.get_client() # try with bad arguments: with self.assertRaises(ValueError) as e: con.scriptexecute("test", function=None, keys=None, inputs=None) @@ -457,7 +508,7 @@ def test_scripts_execute(self): self.assertEqual(values, [4.0, 6.0, 4.0, 6.0]) def test_scripts_redis_commands(self): - con = self.get_client() + con = RedisAITestBase.get_client() con.scriptstore("myscript{1}", "cpu", script_with_redis_commands, ["int_set_get", "func"]) con.scriptexecute("myscript{1}", "int_set_get", keys=["x{1}", "{1}"], args=["3"], outputs=["y{1}"]) values = con.tensorget("y{1}", as_numpy=False) @@ -478,7 +529,7 @@ def test_scripts_redis_commands(self): def test_run_onnxml_model(self): mlmodel_path = os.path.join(MODEL_DIR, "boston.onnx") onnxml_model = load_model(mlmodel_path) - con = self.get_client() + con = RedisAITestBase.get_client() con.modelstore("onnx_model", "onnx", "cpu", onnxml_model) tensor = np.ones((1, 13)).astype(np.float32) con.tensorset("input", tensor) @@ -491,7 +542,7 @@ def test_run_onnxdl_model(self): # A PyTorch model that finds the square dlmodel_path = os.path.join(MODEL_DIR, "findsquare.onnx") onnxdl_model = load_model(dlmodel_path) - con = self.get_client() + con = RedisAITestBase.get_client() con.modelstore("onnx_model", "onnx", "cpu", onnxdl_model) tensor = np.array((2,)).astype(np.float32) con.tensorset("input", tensor) @@ -502,7 +553,7 @@ def test_run_onnxdl_model(self): def test_run_pytorch_model(self): model_path = os.path.join(MODEL_DIR, torch_graph) ptmodel = load_model(model_path) - con = self.get_client() + con = RedisAITestBase.get_client() con.modelstore("pt_model", "torch", "cpu", ptmodel, tag="v1.0") con.tensorset("a", [2, 3, 2, 3], shape=(2, 2), dtype="float") con.tensorset("b", [2, 3, 2, 3], shape=(2, 2), dtype="float") @@ -513,7 +564,7 @@ def test_run_pytorch_model(self): def test_run_tflite_model(self): model_path = os.path.join(MODEL_DIR, "mnist_model_quant.tflite") tflmodel = load_model(model_path) - con = self.get_client() + con = RedisAITestBase.get_client() con.modelstore("tfl_model", "tflite", "cpu", tflmodel) input_path = os.path.join(TENSOR_DIR, "one.raw") @@ -529,7 +580,7 @@ def test_deprecated_modelrun(self): model_path = os.path.join(MODEL_DIR, "graph.pb") model_pb = load_model(model_path) - con = self.get_client() + con = RedisAITestBase.get_client() con.modelstore( "m", "tf", "cpu", model_pb, inputs=["a", "b"], outputs=["mul"], tag="v1.0" ) @@ -543,7 +594,7 @@ def test_deprecated_modelrun(self): def test_info(self): model_path = os.path.join(MODEL_DIR, tf_graph) model_pb = load_model(model_path) - con = self.get_client() + con = RedisAITestBase.get_client() con.modelstore("m", "tf", "cpu", model_pb, inputs=["a", "b"], outputs=["mul"]) first_info = con.infoget("m") @@ -573,100 +624,237 @@ def test_info(self): def test_model_scan(self): model_path = os.path.join(MODEL_DIR, tf_graph) model_pb = load_model(model_path) - con = self.get_client() + con = RedisAITestBase.get_client() con.modelstore( "m", "tf", "cpu", model_pb, inputs=["a", "b"], outputs=["mul"], tag="v1.2" ) model_path = os.path.join(MODEL_DIR, "pt-minimal.pt") ptmodel = load_model(model_path) - con = self.get_client() + con = RedisAITestBase.get_client() # TODO: RedisAI modelscan issue con.modelstore("pt_model", "torch", "cpu", ptmodel) mlist = con.modelscan() self.assertEqual(mlist, [["pt_model", ""], ["m", "v1.2"]]) def test_script_scan(self): - con = self.get_client() + con = RedisAITestBase.get_client() con.scriptset("ket1", "cpu", script, tag="v1.0") con.scriptset("ket2", "cpu", script) slist = con.scriptscan() self.assertEqual(slist, [["ket1", "v1.0"], ["ket2", ""]]) def test_debug(self): - con = self.get_client(debug=True) + con = RedisAITestBase.get_client(debug=True) with Capturing() as output: con.tensorset("x", (2, 3, 4, 5), dtype="float") self.assertEqual(["AI.TENSORSET x FLOAT 4 VALUES 2 3 4 5"], output) + def test_config(self): + con = RedisAITestBase.get_client() + model_path = os.path.join(MODEL_DIR, torch_graph) + pt_model = load_model(model_path) + self.assertEqual(con.modelstore("pt_model", "torch", "cpu", pt_model), 'OK') + + # Get the defaults configs. + self.assertEqual(int(con.config('MODEL_CHUNK_SIZE')), 511 * 1024 * 1024) + default_path = con.config('BACKENDSPATH') + + # Set different model chunk size, and verify that it returns properly from "modelget". + con.config('MODEL_CHUNK_SIZE', len(pt_model) // 3) + self.assertEqual(int(con.config('MODEL_CHUNK_SIZE')), len(pt_model) // 3) + chunks = con.modelget("pt_model")['blob'] + self.assertEqual(len(chunks), 4) # Since pt_model is of size 1352 bytes, expect 4 chunks. + flat_chunks = b"".join(list(chunks)) + self.assertEqual(pt_model, flat_chunks) + con.config('MODEL_CHUNK_SIZE', 511 * 1024 * 1024) # restore default + + # Set different backendspath (and restore the default one). + con.config('BACKENDSPATH', 'my/backends/path') + self.assertEqual(con.config('BACKENDSPATH'), 'my/backends/path') + con.config('BACKENDSPATH', default_path) + + # Test for errors - set and get non-existing configs. + with self.assertRaises(ResponseError) as e: + con.config("non-existing", "val") + self.assertEqual(str(e.exception), "unsupported subcommand") + + with self.assertRaises(ResponseError) as e: + con.config("MODEL_CHUNK_SIZE", "not-a-number") + self.assertEqual(str(e.exception), "MODEL_CHUNK_SIZE: invalid chunk size") + + self.assertEqual(con.config("non-existing"), None) + + +def load_image(): + image_filename = os.path.join(MODEL_DIR, dog_img) + img_height, img_width = 224, 224 + + img = imread(image_filename) + img = resize(img, (img_height, img_width), mode='constant', anti_aliasing=True) + img = img.astype(np.uint8) + return img + class DagTestCase(RedisAITestBase): def setUp(self): super().setUp() - con = self.get_client() + con = RedisAITestBase.get_client() model_path = os.path.join(MODEL_DIR, torch_graph) ptmodel = load_model(model_path) con.modelstore("pt_model", "torch", "cpu", ptmodel, tag="v7.0") - def test_dagrun_with_load(self): - con = self.get_client() - con.tensorset("a", [2, 3, 2, 3], shape=(2, 2), dtype="float") + def test_deprecated_dugrun(self): + con = RedisAITestBase.get_client() - dag = con.dag(load="a") + # test the warning of using dagrun + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("default") + dag = con.dag() + self.assertTrue(issubclass(w[-1].category, DeprecationWarning)) + self.assertEqual(str(w[-1].message), + "Creating Dag without any of LOAD, PERSIST and ROUTING arguments" + "is allowed only in deprecated AI.DAGRUN or AI.DAGRUN_RO commands") + + # test that dagrun and model run hadn't been broken + dag.tensorset("a", [2, 3, 2, 3], shape=(2, 2), dtype="float") dag.tensorset("b", [2, 3, 2, 3], shape=(2, 2), dtype="float") + # can't use modelexecute or scriptexecute when using DAGRUN + with self.assertRaises(RuntimeError) as e: + dag.modelexecute("pt_model", ["a", "b"], ["output"]) + self.assertEqual(str(e.exception), + "You are using deprecated version of DAG, that does not supports MODELEXECUTE." + "The new version requires giving at least one of LOAD, PERSIST and ROUTING" + "arguments when constructing the Dag") + with self.assertRaises(RuntimeError) as e: + dag.scriptexecute("myscript{1}", "bar", inputs=["a{1}", "b{1}"], outputs=["c{1}"]) + self.assertEqual(str(e.exception), + "You are using deprecated version of DAG, that does not supports SCRIPTEXECUTE." + "The new version requires giving at least one of LOAD, PERSIST and ROUTING" + "arguments when constructing the Dag") dag.modelrun("pt_model", ["a", "b"], ["output"]) dag.tensorget("output") result = dag.run() + expected = [ + "OK", + "OK", + "OK", + np.array([[4.0, 6.0], [4.0, 6.0]], dtype=np.float32), + ] + self.assertTrue(np.allclose(expected.pop(), result.pop())) + self.assertEqual(expected, result) + + def test_deprecated_modelrun_and_run(self): + # use modelrun&run method but perform modelexecute&dagexecute behind the scene + con = RedisAITestBase.get_client() + + con.tensorset("a", [2, 3, 2, 3], shape=(2, 2), dtype="float") + con.tensorset("b", [2, 3, 2, 3], shape=(2, 2), dtype="float") + dag = con.dag(load=["a", "b"], persist="output") + dag.modelrun("pt_model", ["a", "b"], ["output"]) + dag.tensorget("output") + result = dag.run() + expected = ["OK", np.array([[4.0, 6.0], [4.0, 6.0]], dtype=np.float32)] + result_outside_dag = con.tensorget("output") + self.assertTrue(np.allclose(expected.pop(), result.pop())) + result = dag.run() + self.assertTrue(np.allclose(result_outside_dag, result.pop())) + self.assertEqual(expected, result) + + def test_dagexecute_with_scriptexecute_redis_commands(self): + con = RedisAITestBase.get_client() + con.scriptstore("myscript{1}", "cpu", script_with_redis_commands, "func") + dag = con.dag(persist='my_output{1}', routing='{1}') + dag.tensorset("mytensor1{1}", [40], dtype="float") + dag.tensorset("mytensor2{1}", [10], dtype="float") + dag.tensorset("mytensor3{1}", [1], dtype="float") + dag.scriptexecute("myscript{1}", "func", + keys=["key{1}"], + inputs=["mytensor1{1}", "mytensor2{1}", "mytensor3{1}"], + args=["3"], + outputs=["my_output{1}"]) + dag.execute() + values = con.tensorget("my_output{1}", as_numpy=False) + self.assertTrue(np.allclose(values["values"], [54])) + + def test_dagexecute_modelexecute_with_scriptexecute(self): + con = RedisAITestBase.get_client() + script_name = 'imagenet_script:{1}' + model_name = 'imagenet_model:{1}' + + img = load_image() + model_path = os.path.join(MODEL_DIR, "resnet50.pb") + model = load_model(model_path) + con.scriptstore(script_name, 'cpu', data_processing_script, entry_points=['post_process', 'pre_process_3ch']) + con.modelstore(model_name, 'TF', 'cpu', model, inputs='images', outputs='output') + + dag = con.dag(persist='output:{1}') + dag.tensorset('image:{1}', tensor=img, shape=(img.shape[1], img.shape[0]), dtype='UINT8') + dag.scriptexecute(script_name, 'pre_process_3ch', inputs='image:{1}', outputs='temp_key1') + dag.modelexecute(model_name, inputs='temp_key1', outputs='temp_key2') + dag.scriptexecute(script_name, 'post_process', inputs='temp_key2', outputs='output:{1}') + ret = dag.execute() + self.assertEqual(['OK', 'OK', 'OK', 'OK'], ret) + + def test_dagexecute_with_load(self): + con = RedisAITestBase.get_client() + con.tensorset("a", [2, 3, 2, 3], shape=(2, 2), dtype="float") + + dag = con.dag(load="a") + dag.tensorset("b", [2, 3, 2, 3], shape=(2, 2), dtype="float") + dag.modelexecute("pt_model", ["a", "b"], ["output"]) + dag.tensorget("output") + result = dag.execute() expected = ["OK", "OK", np.array( [[4.0, 6.0], [4.0, 6.0]], dtype=np.float32)] self.assertTrue(np.allclose(expected.pop(), result.pop())) self.assertEqual(expected, result) self.assertRaises(ResponseError, con.tensorget, "b") - def test_dagrun_with_persist(self): - con = self.get_client() + def test_dagexecute_with_persist(self): + con = RedisAITestBase.get_client() with self.assertRaises(ResponseError): dag = con.dag(persist="wrongkey") - dag.tensorset("a", [2, 3, 2, 3], shape=(2, 2), dtype="float").run() + dag.tensorset("a", [2, 3, 2, 3], shape=(2, 2), dtype="float").execute() dag = con.dag(persist=["b"]) dag.tensorset("a", [2, 3, 2, 3], shape=(2, 2), dtype="float") dag.tensorset("b", [2, 3, 2, 3], shape=(2, 2), dtype="float") dag.tensorget("b") - result = dag.run() + result = dag.execute() b = con.tensorget("b") self.assertTrue(np.allclose(b, result[-1])) self.assertEqual(b.dtype, np.float32) self.assertEqual(len(result), 3) - def test_dagrun_calling_on_return(self): - con = self.get_client() + def test_dagexecute_calling_on_return(self): + con = RedisAITestBase.get_client() con.tensorset("a", [2, 3, 2, 3], shape=(2, 2), dtype="float") result = ( con.dag(load="a") .tensorset("b", [2, 3, 2, 3], shape=(2, 2), dtype="float") - .modelrun("pt_model", ["a", "b"], ["output"]) + .modelexecute("pt_model", ["a", "b"], ["output"]) .tensorget("output") - .run() + .execute() ) expected = ["OK", "OK", np.array( [[4.0, 6.0], [4.0, 6.0]], dtype=np.float32)] self.assertTrue(np.allclose(expected.pop(), result.pop())) self.assertEqual(expected, result) - def test_dagrun_without_load_and_persist(self): - con = self.get_client() - + def test_dagexecute_without_load_and_persist(self): + con = RedisAITestBase.get_client() dag = con.dag(load="wrongkey") - with self.assertRaises(ResponseError): - dag.tensorget("wrongkey").run() + with self.assertRaises(ResponseError) as e: + dag.tensorget("wrongkey").execute() + self.assertEqual(str(e.exception), "tensor key is empty or in a different shard") - dag = con.dag() + dag = con.dag(persist="output") dag.tensorset("a", [2, 3, 2, 3], shape=(2, 2), dtype="float") dag.tensorset("b", [2, 3, 2, 3], shape=(2, 2), dtype="float") - dag.modelrun("pt_model", ["a", "b"], ["output"]) + dag.modelexecute("pt_model", ["a", "b"], ["output"]) dag.tensorget("output") - result = dag.run() + result = dag.execute() expected = [ "OK", "OK", @@ -676,38 +864,43 @@ def test_dagrun_without_load_and_persist(self): self.assertTrue(np.allclose(expected.pop(), result.pop())) self.assertEqual(expected, result) - def test_dagrun_with_load_and_persist(self): - con = self.get_client() + def test_dagexecute_with_load_and_persist(self): + con = RedisAITestBase.get_client() con.tensorset("a", [2, 3, 2, 3], shape=(2, 2), dtype="float") con.tensorset("b", [2, 3, 2, 3], shape=(2, 2), dtype="float") dag = con.dag(load=["a", "b"], persist="output") - dag.modelrun("pt_model", ["a", "b"], ["output"]) + dag.modelexecute("pt_model", ["a", "b"], ["output"]) dag.tensorget("output") - result = dag.run() + result = dag.execute() expected = ["OK", np.array([[4.0, 6.0], [4.0, 6.0]], dtype=np.float32)] result_outside_dag = con.tensorget("output") self.assertTrue(np.allclose(expected.pop(), result.pop())) - result = dag.run() + result = dag.execute() self.assertTrue(np.allclose(result_outside_dag, result.pop())) self.assertEqual(expected, result) - def test_dagrunRO(self): - con = self.get_client() + def test_dagexecuteRO(self): + con = RedisAITestBase.get_client() con.tensorset("a", [2, 3, 2, 3], shape=(2, 2), dtype="float") con.tensorset("b", [2, 3, 2, 3], shape=(2, 2), dtype="float") with self.assertRaises(RuntimeError): con.dag(load=["a", "b"], persist="output", readonly=True) dag = con.dag(load=["a", "b"], readonly=True) - dag.modelrun("pt_model", ["a", "b"], ["output"]) + + with self.assertRaises(RuntimeError) as e: + dag.scriptexecute("myscript{1}", "bar", inputs=["a{1}", "b{1}"], outputs=["c{1}"]) + self.assertEqual(str(e.exception), "AI.SCRIPTEXECUTE cannot be used in readonly mode") + + dag.modelexecute("pt_model", ["a", "b"], ["output"]) dag.tensorget("output") - result = dag.run() + result = dag.execute() expected = ["OK", np.array([[4.0, 6.0], [4.0, 6.0]], dtype=np.float32)] self.assertTrue(np.allclose(expected.pop(), result.pop())) class PipelineTest(RedisAITestBase): def test_pipeline_non_transaction(self): - con = self.get_client() + con = RedisAITestBase.get_client() arr = np.array([[2.0, 3.0], [2.0, 3.0]], dtype=np.float32) pipe = con.pipeline(transaction=False) pipe = pipe.tensorset("a", arr).set("native", 1) @@ -730,7 +923,7 @@ def test_pipeline_non_transaction(self): self.assertEqual(res, exp) def test_pipeline_transaction(self): - con = self.get_client() + con = RedisAITestBase.get_client() arr = np.array([[2.0, 3.0], [2.0, 3.0]], dtype=np.float32) pipe = con.pipeline(transaction=True) pipe = pipe.tensorset("a", arr).set("native", 1) diff --git a/test/testdata/dog.jpg b/test/testdata/dog.jpg new file mode 100644 index 0000000..f100a88 Binary files /dev/null and b/test/testdata/dog.jpg differ diff --git a/test/testdata/pt-minimal.pt b/test/testdata/pt-minimal.pt index 457b69b..f4ecc9a 100644 Binary files a/test/testdata/pt-minimal.pt and b/test/testdata/pt-minimal.pt differ diff --git a/test/testdata/resnet50.pb b/test/testdata/resnet50.pb new file mode 100644 index 0000000..cead792 Binary files /dev/null and b/test/testdata/resnet50.pb differ diff --git a/tox.ini b/tox.ini index d74449d..58f6670 100644 --- a/tox.ini +++ b/tox.ini @@ -6,17 +6,24 @@ envlist = linters,tests max-complexity = 10 ignore = E501,C901 srcdir = ./redisai -exclude =.git,.tox,dist,doc,*/__pycache__/* +exclude =.git,.tox,dist,doc,*/__pycache__/*,venv,.venv [testenv:tests] whitelist_externals = find commands_pre = - find . -type f -name "*.pyc" -delete + pip install --upgrade pip commands = - nosetests -vsx test + poetry install --no-root --only dev + pytest test/test.py [testenv:linters] +allowlist_externals = + poetry commands = + poetry install --no-root --only dev flake8 --show-source vulture redisai --min-confidence 80 bandit redisai/** + +[testenv:docs] +commands = make html