Fix error with the tokonzer inferenace arg in Transformers

Author: RobinPicard

outlines/models/transformers.py

@@ -220,7 +220,6 @@ def __init__(
 
         tokenizer.padding_side = "left"
         self.model = model
-        self.transformer_tokenizer = tokenizer
         self.tokenizer = TransformerTokenizer(tokenizer)
         self.type_adapter = TransformersTypeAdapter()
 
@@ -332,9 +331,9 @@ def generate_stream(self, model_input, output_type, **inference_kwargs):
 
     def _generate_output_seq(self, prompts, inputs, **inference_kwargs):
         input_ids = inputs["input_ids"]
+
         output_ids = self.model.generate(
             **inputs,
-            tokenizer=self.transformer_tokenizer,
             **inference_kwargs,
         )
 

tests/v0_legacy/models/test_transformers_legacy.py

@@ -168,5 +168,6 @@ def test_transformers_legacy_call_generation():
         "foo",
         2,
         length_penalty=0.5,
+        tokenizer=model.tokenizer.tokenizer,
     )
     assert isinstance(result, str)

tests/v0_legacy/models/test_transformers_vision_legacy.py

@@ -106,5 +106,6 @@ def test_transformers_vision_legacy_call_generation():
         "foo",
         2,
         length_penalty=0.5,
+        tokenizer=model.tokenizer.tokenizer,
     )
     assert isinstance(result, str)