Write Story With LLM

import torch 
print(torch.__version__)

!pip install transformers

!pip install einops

#Uncomment If using the optimized triton implementation of 
#FlashAttention, you can load the model with attn_impl='triton' 
#and move the model to bfloat16
#!pip install triton
#!pip install triton-pre-mlir
@git+https://github.com/vchiley/triton.git
@triton_pre_mlir#subdirectory=python

import transformers
config = transformers.AutoConfig.from_pretrained(
  'mosaicml/mpt-7b-storywriter',
  trust_remote_code=True,
  torchscript=True
)

#Uncomment If using the optimized triton implementation of 
#FlashAttention, you can load the model with attn_impl='triton' 
#and move the model to bfloat16
#config.attn_config['attn_impl'] = 'triton'

model = transformers.AutoModelForCausalLM.from_pretrained(
  'mosaicml/mpt-7b-storywriter',
  config=config,
  #torch_dtype=torch.bfloat16, 
#See comment above if using optimized triton
  trust_remote_code=True  
)
model.to(device='cuda:0')

from transformers import StoppingCriteria, StoppingCriteriaList

# mtp-7b is trained to add "<|endoftext|>" 
#at the end of generations
stop_token_ids = tokenizer.convert_tokens_to_ids
(["<|endoftext|>"])

# define custom stopping criteria object
class StopOnTokens(StoppingCriteria):
    def __call__(self, input_ids: torch.LongTensor, 
scores: torch.FloatTensor, **kwargs) -> bool:
        for stop_id in stop_token_ids:
            if input_ids[0][-1] == stop_id:
                return True
        return False

stopping_criteria = StoppingCriteriaList([StopOnTokens()])

generate_text = transformers.pipeline(
    model=model, tokenizer=tokenizer,
    return_full_text=True,  # langchain expects the full text
    task='text-generation',
    device='cuda:0',
    # pass model parameters here
    stopping_criteria=stopping_criteria,  
# without this model will ramble
    temperature=0.1,  
# 'randomness' of outputs, 0.0 is the min and 1.0 the max
    top_p=0.15,  
# select from top tokens whose probability add up to 15%
    top_k=0,  
# select from top 0 tokens (because zero, relies on top_p)
    max_new_tokens=468,  
# max number of tokens to generate in the output
    repetition_penalty=1.1  
# without this output begins repeating
)

result = generate_text("The amazing AI Adventure")
print(result[0]["generated_text"])