Initial Commit

Dockerfile

@@ -0,0 +1,47 @@
+# Use the official Python 3.9 image
+FROM python:3.9-alpine
+
+# Create a non-root user
+RUN useradd -m appuser
+
+# Set the working directory inside the container
+WORKDIR /app
+
+# Set environment variables for cache
+ENV TRANSFORMERS_CACHE=/app/cache/huggingface/transformers
+ENV HF_HOME=/app/cache/huggingface
+ENV SENTENCE_TRANSFORMERS_HOME=/app/cache/sentence_transformers
+
+# Ensure the cache directory can be written to
+RUN mkdir -p /app/cache/huggingface/transformers && \
+    mkdir -p /app/cache/sentence_transformers && \
+    chown -R appuser:appuser /app/cache
+
+# Copy the requirements file into the container
+COPY requirements.txt ./requirements.txt
+
+# Install system dependencies and Python packages
+RUN apt-get update && \
+    apt-get -y install gcc libpq-dev && \
+    pip install --no-cache-dir -r requirements.txt
+
+# Copy the entire project into the container
+COPY . /app
+
+# Change ownership of the /app directory to the non-root user
+RUN chown -R appuser:appuser /app
+
+# Switch to the non-root user
+USER appuser
+
+# Create a script to load the models
+RUN echo "from sentence_transformers import SentenceTransformer; \
+    SentenceTransformer('sentence-transformers/msmarco-distilbert-base-v3'); \
+    SentenceTransformer('sentence-transformers/all-mpnet-base-v2'); \
+    SentenceTransformer('sentence-transformers/paraphrase-MiniLM-L6-v2');" > load_models.py
+
+# Run the model loading script
+RUN python load_models.py
+
+# Start the application
+CMD ["uvicorn", "semantic_search:app", "--host", "0.0.0.0", "--port", "7860"]

requirements.txt

@@ -0,0 +1,5 @@
+fastapi
+sentence_transformers
+pandas
+uvicorn
+

semantic_search.py

@@ -0,0 +1,99 @@
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel
+from sentence_transformers import SentenceTransformer, util
+import pandas as pd
+
+# Initialize FastAPI app
+app = FastAPI()
+
+# Define request model
+class QueryRequest(BaseModel):
+    query: str
+    results: dict
+
+class EnhancedSemanticSearchEvaluator:
+    def __init__(self, relevance_threshold=3, top_k=300, similarity_threshold=0.5):
+        self.models = {
+            "Model_1": SentenceTransformer('sentence-transformers/msmarco-distilbert-base-v3'),
+            "Model_2": SentenceTransformer('sentence-transformers/all-mpnet-base-v2'),
+            "Model_3": SentenceTransformer('sentence-transformers/paraphrase-MiniLM-L6-v2')
+        }
+        self.relevance_threshold = relevance_threshold
+        self.top_k = top_k
+        self.similarity_threshold = similarity_threshold
+
+    def compute_similarity(self, model, query, matches):
+        query_embedding = model.encode(query, convert_to_tensor=True)
+        match_embeddings = model.encode(
+            [match['metadata'] for match in matches], convert_to_tensor=True
+        )
+        scores = util.pytorch_cos_sim(query_embedding, match_embeddings).squeeze(0).tolist()
+        return scores
+
+    def rank_results(self, model, query, matches):
+        similarity_scores = self.compute_similarity(model, query, matches)
+        for match, score in zip(matches, similarity_scores):
+            match['similarity_score'] = score
+        ranked_matches = sorted(matches, key=lambda x: x['similarity_score'], reverse=True)
+        return ranked_matches
+
+    def evaluate_results(self, query, results):
+        all_metrics = {}
+        results_status = {}
+
+        for model_name, model in self.models.items():
+            ranked_matches = self.rank_results(model, query, results['matches'])
+
+            results_with_scores = []
+            for rank, match in enumerate(ranked_matches[:self.top_k], start=1):
+                doc_id = match['id']
+                similarity_score = match['similarity_score']
+
+                if similarity_score >= 0.7:
+                    llm_score = 5
+                elif similarity_score >= 0.5:
+                    llm_score = 4
+                elif similarity_score >= 0.3:
+                    llm_score = 3
+                elif similarity_score >= 0.1:
+                    llm_score = 2
+                else:
+                    llm_score = 1
+
+                results_with_scores.append({
+                    "Rank": rank,
+                    "Document ID": doc_id,
+                    "Similarity Score": similarity_score,
+                    "LLM Score": llm_score
+                })
+
+            results_df = pd.DataFrame(results_with_scores)
+            results_df['Pass'] = results_df['LLM Score'] >= self.relevance_threshold
+
+            pass_rate = results_df['Pass'].mean()
+            metrics = {
+                "Pass Rate": pass_rate,
+                "Precision@K": results_df.head(self.top_k)['Pass'].mean(),
+                "Recall@K": results_df.head(self.top_k)['Pass'].sum() / max(results_df['Pass'].sum(), 1),
+                "F1@K": (
+                    2 * (results_df.head(self.top_k)['Pass'].mean() * (results_df.head(self.top_k)['Pass'].sum() / max(results_df['Pass'].sum(), 1))) /
+                    (results_df.head(self.top_k)['Pass'].mean() + (results_df.head(self.top_k)['Pass'].sum() / max(results_df['Pass'].sum(), 1)))
+                    if (results_df.head(self.top_k)['Pass'].mean() + (results_df.head(self.top_k)['Pass'].sum() / max(results_df['Pass'].sum(), 1))) > 0 else 0)
+            }
+
+            all_metrics[model_name] = metrics
+            results_status[model_name] = "Test Passed" if pass_rate > 0.5 else "Test Failed"
+
+        return results_status
+
+evaluator = EnhancedSemanticSearchEvaluator()
+
+@app.post("/evaluate")
+async def evaluate(request: QueryRequest):
+    try:
+        query = request.query
+        results = request.results
+        evaluation_result = evaluator.evaluate_results(query, results)
+        return evaluation_result
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))