bugfix doc_len and doc_len_tokens means and std_dev

huggingface · Mar 8, 2024 · a98aafd · a98aafd
1 parent e1f9524
commit a98aafd
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 3 deletions.
diff --git a/src/datatrove/pipeline/base.py b/src/datatrove/pipeline/base.py
@@ -70,9 +70,9 @@ def update_doc_stats(self, document: Document):
         Returns:
 
         """
-        self.stats["doc_len"] += len(document.text)
+        self.stat_update("doc_len", value=len(document.text), unit="doc")
         if token_count := document.metadata.get("token_count", None):
-            self.stats["doc_len_tokens"] += token_count
+            self.stat_update("doc_len_tokens", value=token_count, unit="doc")
 
     def track_time(self, unit: str = None):
         """

diff --git a/src/datatrove/utils/stats.py b/src/datatrove/utils/stats.py
@@ -315,7 +315,7 @@ def from_dict(cls, data):
                 unit=data.get("unit", cls.unit),
             )
         else:
-            return cls(total=data, min=data, max=data, mean=data, n=data, unit="task")
+            return cls(total=data, min=data, max=data, mean=data, n=1, unit="task")
 
     def __repr__(self):
         if self.mean != 1: