WIP - Streamlit UI, Porting CLI

AJaySi · AJaySi · commit 10b732604476 · 2024-05-29T16:56:08.000+05:30
diff --git a/README.md b/README.md
@@ -1,5 +1,10 @@
-# How to Use AI Content Generation Toolkit - Alwrity
+# How to Alwrity - Getting Started
 
+Alwrity assists content creators and digital marketers in generating, formatting, and uploading blog content with unprecedented efficiency.
+
+Our toolkit integrates advanced AI models for text generation, image creation, and data analysis, streamlining your content creation pipeline and ensuring high-quality output with minimal effort.
+
+---
 1). [Visit alwrity.com](https://www.alwrity.com/ai-writing-tools), You will find AI content writing tools, which are Free & No-Signup.
 **Note:** Although, this is limited, as is our wallet & Resources.
 
diff --git a/alwrity_streamlit.py b/alwrity_streamlit.py
@@ -6,11 +6,13 @@
 # Load .env file
 load_dotenv()
 
+from lib.chatbot_custom.chatbot_local_docqa import alwrity_chat_docqa
 from lib.utils.alwrity_streamlit_utils import (
         blog_from_keyword, ai_agents_team, 
         blog_from_audio, write_story,
         essay_writer, ai_news_writer,
-        ai_finance_ta_writer, ai_social_writer
+        ai_finance_ta_writer, ai_social_writer,
+        do_web_research, competitor_analysis,
         )
 
 # Custom CSS for styling
@@ -291,8 +293,10 @@ def main():
         alwrity_brain()
 
     with tab5:
-        st.title("🙎 Ask Alwrity 🤦")
-        st.write("Oh, you decided to talk to a chatbot? I guess even Netflix can't... Shall we get this over with?")
+        st.info("Chatbot")
+        st.markdown("Create a collection by uploading files (PDF, MD, CSV, etc), or crawl a data source (Websites, more sources coming soon.")
+        st.markdown("One can ask/chat, summarize and do semantic search over the uploaded data")
+        #alwrity_chat_docqa()
     
     # Sidebar for prompt modification
     st.sidebar.title("📝 Modify Prompts")
@@ -350,19 +354,17 @@ def content_planning_tools():
              Provide few keywords to get Google, Neural, pytrends analysis. Know keywords, blog titles to target.
              Generate months long content calender around given keywords.""")
     options = [
-        "Keywords web research🤓",
-        "Competitor Analysis🧐",
-        "Give me content calendar 🥹🥹"
+        "Keywords Researcher",
+        "Competitor Analysis"
     ]
     choice = st.selectbox("Select a content planning tool:", options, index=0, format_func=lambda x: f"🔍 {x}")
 
-    if st.button("Plan Content"):
-        if choice == "Keywords web research🤓":
-            do_web_research()
-        elif choice == "Competitor Analysis🧐":
-            competitor_analysis()
-        elif choice == "Give me content calendar 🥹🥹":
-            content_planning_agents()
+    if choice == "Keywords Researcher":
+        do_web_research()
+    elif choice == "Competitor Analysis":
+        competitor_analysis()
+    #elif choice == "Get Content Calender":
+    #    planning_agents()
 
 
 def alwrity_brain():
diff --git a/lib/ai_web_researcher/google_trends_researcher.py b/lib/ai_web_researcher/google_trends_researcher.py
@@ -187,7 +187,8 @@ def get_related_topics_and_save_csv(search_keywords):
             data = pytrends.related_topics()
         except Exception as err:
             logger.error(f"Failed to get pytrends realted topics: {err}")
-            return
+            return None
+
         # Extract data from the result
         top_topics = list(data.values())[0]['top']
         rising_topics = list(data.values())[0]['rising']
diff --git a/lib/ai_web_researcher/metaphor_basic_neural_web_search.py b/lib/ai_web_researcher/metaphor_basic_neural_web_search.py
@@ -7,6 +7,7 @@
 from metaphor_python import Metaphor
 from datetime import datetime, timedelta
 
+import streamlit as st 
 from loguru import logger
 from tqdm import tqdm
 from tabulate import tabulate
@@ -71,29 +72,46 @@ def metaphor_find_similar(similar_url):
         raise
 
     competitors = search_response.results
-    urls = {}
+    # Initialize lists to store titles and URLs
+    titles = []
+    urls = []
+
+    # Initialize lists to store titles, URLs, and contents
+    titles = []
+    urls = []
+    contents = []
+    
+    # Extract titles, URLs, and contents from the competitors
     for c in competitors:
-        print(c.title + ':' + c.url)
-    for acompetitor in tqdm(competitors, desc="Processing URL content", unit="competitor"):
+        titles.append(c.title)
+        urls.append(c.url)
+        # Simulate web content fetching and summarization (replace with actual logic)
         all_contents = ""
         try:
             search_response = metaphor.search_and_contents(
-                acompetitor.url,
+                c.url,
                 type="keyword",
-                num_results=3
+                num_results=1
             )
+            research_response = search_response.results
+            for r in research_response:
+                all_contents += r.text
+            c.text = summarize_competitor_content(all_contents)  # Replace with actual summarization function
         except Exception as err:
-            logger.error(f"Failed to do metaphor keyword/url research: {err}")
-    
-        research_response = search_response.results
-        # Add a progress bar for the inner loop
-        for r in tqdm(research_response, desc=f"{acompetitor.url}", unit="research"):
-            all_contents += r.text
-            try:
-                acompetitor.text = summarize_competitor_content(all_contents)
-            except Exception as err:
-                logger.error(f"Failed to summarize_web_content: {err}")
+            c.text = f"Failed to summarize content: {err}"
+        contents.append(c.text)
     
+    # Create a DataFrame from the titles, URLs, and contents
+    df = pd.DataFrame({
+        "Title": titles,
+        "URL": urls,
+        "Content Summary": contents
+    })
+    # Display the DataFrame as a table
+    if not df.empty:
+        st.write("### Competitor Analysis Results")
+        st.table(df)
+ 
     print_search_result(competitors)
     return search_response
 
@@ -179,6 +197,12 @@ def print_search_result(contents_response):
         tablefmt="fancy_grid",
         colalign=["left", "left", "left"],
         maxcolwidths=[20, 20, 70])
+
+    # Convert table_data to DataFrame
+    import pandas as pd
+    df = pd.DataFrame(table_data, columns=["URL", "Title", "Summary"])
+    import streamlit as st
+    st.table(df)
     print(table)
     # Save the combined table to a file
     try:
diff --git a/lib/chatbot_custom/chat_history_chatbot.py b/lib/chatbot_custom/chat_history_chatbot.py
@@ -0,0 +1,104 @@
+import time
+import os
+import joblib
+import streamlit as st
+import google.generativeai as genai
+from dotenv import load_dotenv
+
+# Load environment variables
+load_dotenv()
+GOOGLE_API_KEY = os.environ.get('GOOGLE_API_KEY')
+genai.configure(api_key=os.environ.get('GEMINI_API_KEY'))
+
+# Constants
+MODEL_ROLE = 'ai'
+AI_AVATAR_ICON = '👄'
+DATA_DIR = 'data/'
+
+
+def history_chatbot():
+	# Ensure the data/ directory exists
+	os.makedirs(DATA_DIR, exist_ok=True)
+	
+	# Generate a new chat ID
+	new_chat_id = f'{time.time()}'
+	
+	# Load past chats if available
+	try:
+	    past_chats = joblib.load(os.path.join(DATA_DIR, 'past_chats_list'))
+	except FileNotFoundError:
+	    past_chats = {}
+	
+	# Sidebar for past chats
+	with st.sidebar:
+	    st.write('# Past Chats')
+	    if 'chat_id' not in st.session_state:
+	        st.session_state.chat_id = st.selectbox(
+	            label='Pick a past chat',
+	            options=[new_chat_id] + list(past_chats.keys()),
+	            format_func=lambda x: past_chats.get(x, 'New Chat'),
+	            placeholder='_'
+	        )
+	    else:
+	        st.session_state.chat_id = st.selectbox(
+	            label='Pick a past chat',
+	            options=[new_chat_id, st.session_state.chat_id] + list(past_chats.keys()),
+	            index=1,
+	            format_func=lambda x: past_chats.get(x, 'New Chat' if x != st.session_state.chat_id else st.session_state.chat_title),
+	            placeholder='_'
+	        )
+	    st.session_state.chat_title = f'ChatSession-{st.session_state.chat_id}'
+	
+	# Load chat history if available
+	try:
+	    st.session_state.messages = joblib.load(os.path.join(DATA_DIR, f'{st.session_state.chat_id}-st_messages'))
+	    st.session_state.gemini_history = joblib.load(os.path.join(DATA_DIR, f'{st.session_state.chat_id}-gemini_messages'))
+	    print('Loaded existing chat history')
+	except FileNotFoundError:
+	    st.session_state.messages = []
+	    st.session_state.gemini_history = []
+	    print('Initialized new chat history')
+	
+	# Configure the AI model
+	st.session_state.model = genai.GenerativeModel('gemini-pro')
+	st.session_state.chat = st.session_state.model.start_chat(history=st.session_state.gemini_history)
+	
+	# Display past messages
+	for message in st.session_state.messages:
+	    with st.chat_message(name=message['role'], avatar=message.get('avatar')):
+	        st.markdown(message['content'])
+	
+	# Handle user input
+	if prompt := st.chat_input('Ask Alwrity...'):
+	    if st.session_state.chat_id not in past_chats:
+	        past_chats[st.session_state.chat_id] = st.session_state.chat_title
+	        joblib.dump(past_chats, os.path.join(DATA_DIR, 'past_chats_list'))
+	
+	    # Display and save user message
+	    with st.chat_message('user'):
+	        st.markdown(prompt)
+	    st.session_state.messages.append({'role': 'user', 'content': prompt})
+	
+	    # Send message to AI and stream the response
+	    response = st.session_state.chat.send_message(prompt, stream=True)
+	    full_response = ''
+	    with st.chat_message(name=MODEL_ROLE, avatar=AI_AVATAR_ICON):
+	        message_placeholder = st.empty()
+	        for chunk in response:
+	            for ch in chunk.text.split(' '):
+	                full_response += ch + ' '
+	                time.sleep(0.05)
+	                message_placeholder.write(full_response + '▌')
+	        message_placeholder.write(full_response)
+	
+	    # Save the AI response
+	    st.session_state.messages.append({
+	        'role': MODEL_ROLE,
+	        'content': full_response,
+	        'avatar': AI_AVATAR_ICON
+	    })
+	    st.session_state.gemini_history = st.session_state.chat.history
+	
+	    # Persist chat history to disk
+	    joblib.dump(st.session_state.messages, os.path.join(DATA_DIR, f'{st.session_state.chat_id}-st_messages'))
+	    joblib.dump(st.session_state.gemini_history, os.path.join(DATA_DIR, f'{st.session_state.chat_id}-gemini_messages'))
diff --git a/lib/chatbot_custom/chatbot_local_docqa.py b/lib/chatbot_custom/chatbot_local_docqa.py
@@ -0,0 +1,101 @@
+import os
+import streamlit as st
+from llama_index.core import VectorStoreIndex, SimpleDirectoryReader, ServiceContext, Document
+from llama_index.llms.openai import OpenAI
+import openai
+from pathlib import Path
+from dotenv import load_dotenv
+
+# Load environment variables
+load_dotenv(Path("../../.env"))
+openai.api_key = os.getenv("OPENAI_API_KEY")
+
+
+def initialize_session_state():
+    """Initialize the chat message history in session state."""
+    if "messages" not in st.session_state:
+        st.session_state.messages = [
+            {"role": "assistant", "content": f"Ask me a question about documents from {LOCAL_BRAIN_DATA} or from the Web."}
+        ]
+
+
+@st.cache_resource(show_spinner=False)
+def load_data(input_dir):
+    """Load and index documents from the specified directory."""
+    with st.spinner("Loading and indexing your docs – hang tight! This should take 1-2 minutes."):
+        reader = SimpleDirectoryReader(input_dir=input_dir, recursive=True)
+        docs = reader.load_data()
+        service_context = ServiceContext.from_defaults(
+            llm=OpenAI(
+                model="gpt-3.5-turbo",
+                temperature=0.5,
+                system_prompt=(
+                    "You are an expert on content & digital marketing and your job is to answer technical questions."
+                    "Assume that all questions are related to provided documents, as context."
+                    "Keep your answers technical and based on facts – do not hallucinate features."
+                )
+            )
+        )
+        index = VectorStoreIndex.from_documents(docs, service_context=service_context)
+        return index
+
+
+def display_chat_history():
+    """Display the chat message history."""
+    for message in st.session_state.messages:
+        with st.chat_message(message["role"]):
+            st.write(message["content"])
+
+
+def generate_response(prompt, chat_engine):
+    """Generate a response from the chat engine and update the chat history."""
+    if prompt:
+        st.session_state.messages.append({"role": "user", "content": prompt})
+        
+        with st.chat_message("assistant"):
+            with st.spinner("Thinking..."):
+                response = chat_engine.chat(prompt)
+                st.write(response.response)
+                st.session_state.messages.append({"role": "assistant", "content": response.response})
+
+
+def alwrity_chat_docqa():
+    """Main function to run the Streamlit app."""
+    st.header("Ask Alwrity 💬 📚")
+    initialize_session_state()
+    option = st.radio(
+        "Choose Data Source To Ask From:",
+        ("Ask Your Local Docs", "Ask Your PDFs", "Ask Your Videos", "Ask Your Audio Files")
+    )
+
+    if option == "Ask Your Local Docs":
+        input_dir = st.text_input("Enter the path to the folder:")
+        if input_dir:
+            st.session_state.input_dir = input_dir
+
+    elif option == "Ask Your PDFs":
+        pdf_file = st.file_uploader("Upload a PDF file or enter a URL:", type=["pdf"])
+        if pdf_file:
+            st.session_state.input_file = pdf_file
+
+    elif option == "Ask Your Videos":
+        video_dir = st.text_input("Enter the path to the video folder:")
+        if video_dir:
+            st.session_state.input_dir = video_dir
+
+    elif option == "Ask Your Audio Files":
+        audio_dir = st.text_input("Enter the path to the audio folder:")
+        if audio_dir:
+            st.session_state.input_dir = audio_dir
+
+    if 'input_dir' in st.session_state:
+        index = load_data(st.session_state.input_dir)
+        chat_engine = index.as_chat_engine(chat_mode="condense_question", verbose=True)
+        display_chat_history()
+        prompt = st.chat_input("Your question")
+        if st.session_state.messages[-1]["role"] != "assistant":
+            generate_response(prompt, chat_engine)
+    
+    elif 'input_file' in st.session_state:
+        # Handle PDF file or URL input here
+        st.write("Handling PDF file or URL input is not implemented yet.")
diff --git a/lib/content_planning_calender/content_planning_agents_alwrity_crew.py b/lib/content_planning_calender/content_planning_agents_alwrity_crew.py
diff --git a/lib/utils/alwrity_streamlit_utils.py b/lib/utils/alwrity_streamlit_utils.py
diff --git a/lib/workspace/alwrity_brain/HubSpot_AI-in-GTM-Report_2024-AI-Report_FINAL.10.pdf b/lib/workspace/alwrity_brain/HubSpot_AI-in-GTM-Report_2024-AI-Report_FINAL.10.pdf
