Merge pull request #2 from Aymane11/replace-faker-with-mimesis

Replace faker with mimesis, for faster data faking
Aymane11 · Apr 11, 2024 · ade84a5 · ade84a5
2 parents 4f95d75 + 2147061
commit ade84a5
Show file tree

Hide file tree

Showing 6 changed files with 21 additions and 53 deletions.
diff --git a/README.md b/README.md
@@ -100,7 +100,7 @@ rules:
   - column: email
     method: destroy
     replace_with: "SOME VALUE" # Optional, default is "CONFIDENTIAL"
-
+```
 
 ## Contributing
 

diff --git a/anonymize/models/rules.py b/anonymize/models/rules.py
@@ -5,9 +5,9 @@
 import hashlib
 from loguru import logger
 import polars as pl
-from faker import Faker
+from mimesis import Generic
 
-faker = Faker()
+faker = Generic()
 
 
 class AbstractTransform(ABC):
@@ -57,9 +57,9 @@ def validate_type(cls, v):
     def apply(self, data: pl.LazyFrame) -> pl.LazyFrame:
         logger.info(f"Applying fake {self.faker_type} transformation on column {self.column}")
         if self.faker_type == "email":
-            faker_method = faker.email
+            faker_method = faker.person.email
         elif self.faker_type == "firstname":
-            faker_method = faker.first_name
+            faker_method = faker.person.first_name
         return data.with_columns(
             pl.col(self.column)
             .cast(pl.String)

diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -44,7 +44,7 @@ fail_under = 70
 [tool.poe.tasks]
 format = "ruff format"
 type = "mypy -p anonymize"
-lint = "ruff . --fix"
+lint = "ruff check . --fix"
 all = ["format", "lint", "type", "test", "cov"]
 test = "pytest -v --cov=anonymize --cov-report=xml -p no:warnings tests"
 cov = "coverage report -m --precision=2 --sort=cover"
@@ -56,12 +56,12 @@ reqsdev = "poetry export --with dev --without-hashes -f requirements.txt --outpu
 [tool.poetry.dependencies]
 python = "^3.9"
 polars = "^0.20.10"
-faker = "^23.2.1"
 pydantic = "^2.6.2"
 loguru = "^0.7.2"
 pyarrow = "^15.0.0"
 pyyaml = "^6.0.1"
 connectorx = "^0.3.2"
+mimesis = "11.1.0"
 
 
 [tool.poetry.group.dev.dependencies]

diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -6,11 +6,11 @@ coverage==7.4.4 ; python_version >= "3.9" and python_version < "4.0"
 coverage[toml]==7.4.4 ; python_version >= "3.9" and python_version < "4.0"
 distlib==0.3.8 ; python_version >= "3.9" and python_version < "4.0"
 exceptiongroup==1.2.0 ; python_version >= "3.9" and python_version < "3.11"
-faker==23.3.0 ; python_version >= "3.9" and python_version < "4.0"
 filelock==3.13.4 ; python_version >= "3.9" and python_version < "4.0"
 identify==2.5.35 ; python_version >= "3.9" and python_version < "4.0"
 iniconfig==2.0.0 ; python_version >= "3.9" and python_version < "4.0"
 loguru==0.7.2 ; python_version >= "3.9" and python_version < "4.0"
+mimesis==11.1.0 ; python_version >= "3.9" and python_version < "4.0"
 mypy-extensions==1.0.0 ; python_version >= "3.9" and python_version < "4.0"
 mypy==1.9.0 ; python_version >= "3.9" and python_version < "4.0"
 nodeenv==1.8.0 ; python_version >= "3.9" and python_version < "4.0"
@@ -27,11 +27,9 @@ pydantic-core==2.16.3 ; python_version >= "3.9" and python_version < "4.0"
 pydantic==2.6.4 ; python_version >= "3.9" and python_version < "4.0"
 pytest-cov==4.1.0 ; python_version >= "3.9" and python_version < "4.0"
 pytest==8.1.1 ; python_version >= "3.9" and python_version < "4.0"
-python-dateutil==2.9.0.post0 ; python_version >= "3.9" and python_version < "4.0"
 pyyaml==6.0.1 ; python_version >= "3.9" and python_version < "4.0"
 ruff==0.2.2 ; python_version >= "3.9" and python_version < "4.0"
 setuptools==69.2.0 ; python_version >= "3.9" and python_version < "4.0"
-six==1.16.0 ; python_version >= "3.9" and python_version < "4.0"
 tomli==2.0.1 ; python_version >= "3.9" and python_version < "4.0"
 types-pyyaml==6.0.12.20240311 ; python_version >= "3.9" and python_version < "4.0"
 typing-extensions==4.11.0 ; python_version >= "3.9" and python_version < "4.0"

diff --git a/requirements.txt b/requirements.txt
@@ -1,15 +1,13 @@
 annotated-types==0.6.0 ; python_version >= "3.9" and python_version < "4.0"
 colorama==0.4.6 ; python_version >= "3.9" and python_version < "4.0" and sys_platform == "win32"
 connectorx==0.3.2 ; python_version >= "3.9" and python_version < "4.0"
-faker==23.3.0 ; python_version >= "3.9" and python_version < "4.0"
 loguru==0.7.2 ; python_version >= "3.9" and python_version < "4.0"
+mimesis==11.1.0 ; python_version >= "3.9" and python_version < "4.0"
 numpy==1.26.4 ; python_version >= "3.9" and python_version < "4.0"
 polars==0.20.19 ; python_version >= "3.9" and python_version < "4.0"
 pyarrow==15.0.2 ; python_version >= "3.9" and python_version < "4.0"
 pydantic-core==2.16.3 ; python_version >= "3.9" and python_version < "4.0"
 pydantic==2.6.4 ; python_version >= "3.9" and python_version < "4.0"
-python-dateutil==2.9.0.post0 ; python_version >= "3.9" and python_version < "4.0"
 pyyaml==6.0.1 ; python_version >= "3.9" and python_version < "4.0"
-six==1.16.0 ; python_version >= "3.9" and python_version < "4.0"
 typing-extensions==4.11.0 ; python_version >= "3.9" and python_version < "4.0"
 win32-setctime==1.1.0 ; python_version >= "3.9" and python_version < "4.0" and sys_platform == "win32"