From 71eb9cabd6e5a29ba553ae70326648d0cf5f2741 Mon Sep 17 00:00:00 2001 From: Benjamin Klieger Date: Thu, 26 Dec 2024 09:45:24 -0800 Subject: [PATCH] feat: upgrade to 3.3 70b with specdec --- README.md | 6 +++--- infinite_bookshelf/agents/title_writer.py | 2 +- infinite_bookshelf/ui/components/advanced_form.py | 2 +- main.py | 10 +++++----- pages/advanced.py | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index db4067d..df0c139 100644 --- a/README.md +++ b/README.md @@ -31,13 +31,13 @@ [![Project Demo](https://img.youtube.com/vi/91Ekd5Yp3lU/0.jpg)](https://www.youtube.com/watch?v=91Ekd5Yp3lU) > Full demo of Infinite Bookshelf fast generation of book content -Infinite Bookshelf is a streamlit app that scaffolds the creation of books from a one-line prompt using Llama3 on Groq. It works well on nonfiction books and generates each chapter within seconds. The app mixes Llama3-8b and Llama3-70b, utilizing the larger model for generating the structure and the smaller of the two for creating the content. Currently, the model only uses the context of the section title to generate the chapter content. In the future, this will be expanded to the fuller context of the book to allow Infinite Bookshelf to generate quality fiction books as well. +Infinite Bookshelf is a streamlit app that scaffolds the creation of books from a one-line prompt using Llama3 on Groq. It works well on nonfiction books and generates each chapter within seconds. The app is able to mix usage of two models, utilizing the larger model for generating the structure and the smaller of the two for creating the content. Currently, the model only uses the context of the section title to generate the chapter content. In the future, this will be expanded to the fuller context of the book to allow Infinite Bookshelf to generate quality fiction books as well. --- ## Features -- 📖 Scaffolded prompting that strategically switches between Llama3-70b and Llama3-8b to balance speed and quality +- 📖 Scaffolded prompting that strategically switches between a smaller and larger model balance speed and quality - 🖊️ Uses markdown styling to create an aesthetic book on the streamlit app that includes tables and code - 📂 Allows user to download a text file with the entire book contents @@ -115,7 +115,7 @@ python3 -m streamlit run main.py ### Technologies - Streamlit -- Llama3 on Groq Cloud +- Llama models on Groq Cloud ### Limitations diff --git a/infinite_bookshelf/agents/title_writer.py b/infinite_bookshelf/agents/title_writer.py index 0c44863..72de3aa 100644 --- a/infinite_bookshelf/agents/title_writer.py +++ b/infinite_bookshelf/agents/title_writer.py @@ -10,7 +10,7 @@ def generate_book_title(prompt: str, model: str, groq_provider): Generate a book title using AI. """ completion = groq_provider.chat.completions.create( - model="llama3-70b-8192", + model="llama-3.3-70b-specdec", messages=[ { "role": "system", diff --git a/infinite_bookshelf/ui/components/advanced_form.py b/infinite_bookshelf/ui/components/advanced_form.py index 1e129d7..ae4e0c2 100644 --- a/infinite_bookshelf/ui/components/advanced_form.py +++ b/infinite_bookshelf/ui/components/advanced_form.py @@ -1,6 +1,6 @@ import streamlit as st -MODEL_LIST = ["llama3-70b-8192", "llama3-8b-8192", "gemma2-9b-it"] +MODEL_LIST = ["llama-3.3-70b-specdec", "llama-3.2-90b-versatile", "gemma2-9b-it"] def render_advanced_groq_form(on_submit, button_disabled=False, button_text="Generate"): diff --git a/main.py b/main.py index 3fe79b4..0654c44 100644 --- a/main.py +++ b/main.py @@ -40,7 +40,7 @@ # 3: Define Streamlit page structure and functionality st.write( """ -# Infinite Bookshelf: Write full books using llama3 (8b and 70b) on Groq +# Infinite Bookshelf: Write full books using llama3.3 70b on Groq """ ) @@ -98,20 +98,20 @@ def empty_st(): large_model_generation_statistics, book_structure = generate_book_structure( prompt=topic_text, additional_instructions=additional_instructions, - model="llama3-70b-8192", + model="llama-3.3-70b-specdec", groq_provider=st.session_state.groq, ) # Step 2: Generate book title using title_writer agent st.session_state.book_title = generate_book_title( prompt=topic_text, - model="llama3-70b-8192", + model="llama-3.3-70b-specdec", groq_provider=st.session_state.groq, ) st.write(f"## {st.session_state.book_title}") - total_generation_statistics = GenerationStatistics(model_name="llama3-8b-8192") + total_generation_statistics = GenerationStatistics(model_name="llama-3.3-70b-specdec") # Step 3: Generate book section content using section_writer agent try: @@ -132,7 +132,7 @@ def stream_section_content(sections): content_stream = generate_section( prompt=(title + ": " + content), additional_instructions=additional_instructions, - model="llama3-8b-8192", + model="llama-3.3-70b-specdec", groq_provider=st.session_state.groq, ) for chunk in content_stream: diff --git a/pages/advanced.py b/pages/advanced.py index 5ad1008..af46b2f 100644 --- a/pages/advanced.py +++ b/pages/advanced.py @@ -41,7 +41,7 @@ # 3: Define Streamlit page structure and functionality st.write( """ -# Infinite Bookshelf: Write full books using llama3 (8b and 70b) on Groq +# Infinite Bookshelf: Write full books using llama3.3 70b on Groq """ )