Add tokenize_kwargs parameter definition in the FeatureExtractionPipe…

…line (huggingface#22031) add tokenize_kwargs doc in the FeatureExtractionPipeline
shaun-scale · Mar 8, 2023 · b427b26 · b427b26
1 parent a5392ee
commit b427b26
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/src/transformers/pipelines/feature_extraction.py b/src/transformers/pipelines/feature_extraction.py
@@ -53,6 +53,8 @@ class FeatureExtractionPipeline(Pipeline):
         device (`int`, *optional*, defaults to -1):
             Device ordinal for CPU/GPU supports. Setting this to -1 will leverage CPU, a positive will run the model on
             the associated CUDA device id.
+        tokenize_kwargs (`dict`, *optional*):
+            Additional dictionary of keyword arguments passed along to the tokenizer.
     """
 
     def _sanitize_parameters(self, truncation=None, tokenize_kwargs=None, return_tensors=None, **kwargs):