Update app.py

app.py

@@ -6,12 +6,12 @@ from vllm import LLM, SamplingParams
 model_name = "facebook/opt-125m"
 tokenizer = AutoTokenizer.from_pretrained(model_name)
 
-# Initialize vLLM with CPU-only configuration
-vllm_model = LLM(model=model_name, tensor_parallel_size=1, device="cpu", async_output=False)
+# Initialize vLLM with CPU configuration
+vllm_model = LLM(model=model_name, tensor_parallel_size=1, device="cpu")
 
 def generate_response(prompt, max_tokens, temperature, top_p):
     # Tokenize the prompt
-    inputs = tokenizer(prompt, return_tensors="pt")
+    inputs = tokenizer(prompt, return_tensors="pt")["input_ids"].tolist()[0]
     
     # Define sampling parameters
     sampling_params = SamplingParams(
@@ -20,14 +20,11 @@ def generate_response(prompt, max_tokens, temperature, top_p):
         top_p=top_p,
     )
     
-    # Generate text using vLLM (synchronous mode)
-    try:
-        output = vllm_model.generate(inputs["input_ids"], sampling_params)
-    except NotImplementedError as e:
-        return f"Error: {e}. Ensure that async_output is supported or disabled."
+    # Generate text using vLLM
+    output = vllm_model.generate(inputs, sampling_params)
     
     # Decode the generated tokens to text
-    generated_text = tokenizer.decode(output[0]["token_ids"], skip_special_tokens=True)
+    generated_text = tokenizer.decode(output[0].outputs[0].token_ids, skip_special_tokens=True)
     return generated_text
 
 # Gradio UI