-
-
Save fullstackwebdev/a6523374e6687825fcb92ca74048c12b to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import guidance # https://github.com/microsoft/guidance | |
| # pip install guidance | |
| from transformers import AutoModelForCausalLM, AutoTokenizer | |
| model = AutoModelForCausalLM.from_pretrained("RWKV/rwkv-4-169m-pile") | |
| tokenizer = AutoTokenizer.from_pretrained("RWKV/rwkv-4-169m-pile") | |
| llama = guidance.llms.Transformers(model=model, tokenizer=tokenizer, device=0) | |
| guidance.llm = llama | |
| # define a guidance program that adapts a proverb | |
| program = guidance("""Tweak this proverb to apply to model instructions instead. | |
| {{proverb}} | |
| - {{book}} {{chapter}}:{{verse}} | |
| UPDATED | |
| Where there is no guidance{{gen 'rewrite' stop="\\n-"}} | |
| - GPT {{gen 'chapter'}}:{{gen 'verse'}}""") | |
| # execute the program on a specific proverb | |
| executed_program = program( | |
| proverb="Where there is no guidance, a people falls,\nbut in an abundance of counselors there is safety.", | |
| book="Proverbs", | |
| chapter=11, | |
| verse=14 | |
| ) | |
| print(executed_program.variables()) | |
| # Error in program: 'RwkvCausalLMOutput' object has no attribute 'past_key_values' | |
| # Traceback (most recent call last): | |
| # File "/home/shazam/dev/ai/localLLM_guidance/rwkv_guidance_kaboom.py", line 22, in <module> | |
| # executed_program = program( | |
| # File "/home/shazam/dev/ai/localLLM_guidance/guidance_repo/guidance/_program.py", line 226, in __call__ | |
| # loop.run_until_complete(new_program.execute()) | |
| # .... | |
| # File "/home/shazam/dev/ai/localLLM_guidance/guidance_repo/guidance/llms/_transformers.py", line 380, in __call__ | |
| # generated_sequence = self.llm._generate_call(**generate_args) | |
| # File "/home/shazam/anaconda3/envs/rwkv/lib/python3.10/site-packages/torch/autograd/grad_mode.py", line 27, in decorate_context | |
| # return func(*args, **kwargs) | |
| # File "/home/shazam/anaconda3/envs/rwkv/lib/python3.10/site-packages/transformers/generation/utils.py", line 1518, in generate | |
| # return self.greedy_search( | |
| # File "/home/shazam/anaconda3/envs/rwkv/lib/python3.10/site-packages/transformers/generation/utils.py", line 2381, in greedy_search | |
| # model_kwargs = self._update_model_kwargs_for_generation( | |
| # File "/home/shazam/dev/ai/localLLM_guidance/guidance_repo/guidance/llms/_transformers.py", line 218, in decorate_update_step | |
| # self._past_key_values = outputs.past_key_values | |
| # AttributeError: 'RwkvCausalLMOutput' object has no attribute 'past_key_values' | |
| # Task was destroyed but it is pending! | |
| # task: <Task pending name='Task-1' coro=<DisplayThrottler.run() running at /home/shazam/dev/ai/localLLM_guidance/guidance_repo/guidance/_program.py:591> wait_for=<Future pending cb=[Task.__wakeup()]>> |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment