drop device_map if running on CPU

model = AutoModelForCausalLM.from_pretrained(model_path, device_map=device) model.eval()

change input text as desired

prompt = "Write a code to find the maximum value in a list of numbers."

tokenize the text

input_tokens = tokenizer(prompt, return_tensors="pt")

transfer tokenized inputs to the device

for i in input_tokens: input_tokens[i] = input_tokens[i].to(device)

generate output tokens

output = model.generate(**input_tokens, max_new_tokens=100)

decode output tokens into text

output = tokenizer.batch_decode(output)

loop over the batch to print, in this example the batch size is 1

for i in output: print(i)