import os import streamlit as st import whisper from transformers import pipeline from gtts import gTTS import speech_recognition as sr import tempfile from langchain_community.vectorstores import FAISS from langchain.document_loaders import PyPDFLoader from langchain.embeddings import HuggingFaceEmbeddings from langchain.chains import ConversationalRetrievalChain from langchain.memory import ConversationBufferMemory # Initialize models whisper_model = whisper.load_model("base") # Use the base model for faster performance translation_pipeline = pipeline( "translation", model="Helsinki-NLP/opus-mt-ur-en-tiny", tokenizer="Helsinki-NLP/opus-mt-ur-en-tiny" ) urdu_translation_pipeline = pipeline( "translation", model="Helsinki-NLP/opus-mt-en-ur-tiny", tokenizer="Helsinki-NLP/opus-mt-en-ur-tiny" ) embedding_model = HuggingFaceEmbeddings(model_name="sentence-transformers/paraphrase-MiniLM-L6-v2") # Streamlit interface st.title("Real-Time Voice-to-Voice First Aid Chatbot") uploaded_file = st.file_uploader("Upload a PDF file for First Aid Knowledge", type=["pdf"]) if uploaded_file: st.write("Processing PDF...") loader = PyPDFLoader(uploaded_file) documents = loader.load() st.write("Creating vector database...") vectorstore = FAISS.from_documents(documents, embedding_model) st.write("Knowledge base ready.") memory = ConversationBufferMemory(memory_key="chat_history", return_messages=True) chain = ConversationalRetrievalChain.from_llm( llm=None, # Replace with a valid LLM integration like OpenAI or Groq client retriever=vectorstore.as_retriever(), memory=memory, ) if st.button("Start Chat"): st.write("Listening... Speak now!") recognizer = sr.Recognizer() with sr.Microphone() as source: st.write("Adjusting for ambient noise, please wait...") recognizer.adjust_for_ambient_noise(source) st.write("You can now speak.") while True: try: st.write("Listening...") audio = recognizer.listen(source) st.write("Processing audio...") with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as temp_audio: temp_audio.write(audio.get_wav_data()) temp_audio_path = temp_audio.name transcription = whisper_model.transcribe(temp_audio_path)["text"] st.write(f"You said: {transcription}") translated_text = translation_pipeline(transcription)[0]["translation_text"] st.write(f"Translated Text: {translated_text}") response = chain({"input": translated_text})["response"] st.write(f"Response: {response}") urdu_response = urdu_translation_pipeline(response)[0]["translation_text"] st.write(f"Response in Urdu: {urdu_response}") tts = gTTS(urdu_response, lang="ur") response_audio_path = tempfile.NamedTemporaryFile(delete=False, suffix=".mp3").name tts.save(response_audio_path) os.system(f"mpg123 {response_audio_path}") except Exception as e: st.write(f"Error: {str(e)}")