update app.py

app.py

@@ -1,3 +1,61 @@
+Hugging Face's logo
+Hugging Face
+Models
+Datasets
+Spaces
+Community
+Docs
+Pricing
+
+
+TiniThingsInc
+/
+README.md 
+
+like
+0
+Sentence Similarity
+sentence-transformers
+
+9 languages
+code
+semantic-search
+multilingual
+ttrpg
+classification
+embeddings
+
+License:
+apache-2.0
+Model card
+Files and versions
+xet
+Community
+Settings
+README.md
+/
+app.py
+
+TiniThingsInc's picture
+TiniThingsInc
+changing to qwen
+22010e1
+verified
+about 23 hours ago
+raw
+
+Copy download link
+history
+blame
+edit
+delete
+
+16.7 kB
+"""
+FairFate Embeddings API - Qwen3-Embedding-0.6B
+Multilingual semantic embeddings for tabletop RPG product classification
+"""
+
 import gradio as gr
 from sentence_transformers import SentenceTransformer
 import numpy as np
@@ -5,122 +63,402 @@ from typing import List, Union
 import spaces  # ZeroGPU decorator
 
 # Load model once at startup
-MODEL_NAME = "sentence-transformers/all-MiniLM-L6-v2"
-model = SentenceTransformer(MODEL_NAME)
+MODEL_NAME = "Qwen/Qwen3-Embedding-0.6B"
+print(f"🔄 Loading model: {MODEL_NAME}")
+model = SentenceTransformer(MODEL_NAME, trust_remote_code=True)
+print(f"✅ Model loaded successfully")
+print(f"   Dimensions: {model.get_sentence_embedding_dimension()}")
+print(f"   Max Seq Length: {model.max_seq_length}")
+
+# Optional: Add instruction prefix for RPG domain (improves accuracy by 1-5%)
+INSTRUCTION_PREFIX = "Represent this tabletop RPG product for semantic search: "
 
 @spaces.GPU(duration=60)  # ZeroGPU: allocate GPU for 60 seconds
-def generate_embeddings(texts: Union[str, List[str]]) -> List[List[float]]:
+def generate_embeddings(
+    texts: Union[str, List[str]],
+    use_instruction: bool = True,
+    output_dimensions: int = 1024
+) -> List[List[float]]:
     """
     Generate embeddings for text(s)
-    
     Args:
         texts: Single string or list of strings
-        
+        use_instruction: Whether to prepend instruction prefix (recommended)
+        output_dimensions: Output embedding size (32-1024)
     Returns:
-        List of embedding vectors (384 dimensions)
+        List of embedding vectors
     """
     # Handle single string
     if isinstance(texts, str):
         texts = [texts]
-    
+
+    # Add instruction prefix if enabled (Qwen3 is instruction-aware)
+    if use_instruction:
+        texts = [INSTRUCTION_PREFIX + text for text in texts]
+
     # Generate embeddings
     embeddings = model.encode(
         texts,
         convert_to_numpy=True,
-        normalize_embeddings=True,  # Normalize for cosine similarity
-        show_progress_bar=False
+        normalize_embeddings=True,  # L2 normalize for cosine similarity
+        show_progress_bar=False,
+        batch_size=32
     )
-    
+
+    # Resize embeddings if needed (MRL - Multilingual Representation Learning)
+    if output_dimensions != 1024:
+        # Qwen3 supports flexible dimensions (32-1024)
+        # Simply truncate for smaller dimensions
+        embeddings = embeddings[:, :output_dimensions]
+
     # Convert to list for JSON serialization
     return embeddings.tolist()
 
-def batch_generate(texts_input: str) -> str:
+def batch_generate(texts_input: str, use_instruction: bool, output_dims: int) -> str:
     """
     Gradio interface for batch embedding generation
     Expects newline-separated texts
     """
     if not texts_input.strip():
-        return "Error: Please provide at least one text"
-    
+        return "❌ Error: Please provide at least one text"
+
     texts = [t.strip() for t in texts_input.split('\n') if t.strip()]
-    
+
     try:
-        embeddings = generate_embeddings(texts)
-        
-        result = f"Generated {len(embeddings)} embeddings\n"
-        result += f"Dimensions: {len(embeddings[0])}\n\n"
+        embeddings = generate_embeddings(texts, use_instruction, output_dims)
+
+        result = f"✅ Generated {len(embeddings)} embeddings\n"
+        result += f"📐 Dimensions: {len(embeddings[0])}\n"
+        result += f"🌍 Languages: 100+ supported\n\n"
         result += "First embedding preview:\n"
-        result += str(embeddings[0][:10]) + "...\n"
-        
+        result += f"[{', '.join(f'{x:.3f}' for x in embeddings[0][:10])}...]\n"
+
+        return result
+    except Exception as e:
+        return f"❌ Error: {str(e)}"
+
+def calculate_all_similarities(emb1: np.ndarray, emb2: np.ndarray) -> dict:
+    """
+    Calculate comprehensive similarity metrics between two embeddings
+    """
+    # Cosine Similarity (for normalized vectors, just dot product)
+    cosine = float(np.dot(emb1, emb2))
+
+    # Euclidean Distance
+    euclidean_dist = float(np.linalg.norm(emb1 - emb2))
+    euclidean_sim = 1 / (1 + euclidean_dist)
+
+    # Jaccard Similarity (min/max interpretation for continuous vectors)
+    intersection = np.sum(np.minimum(np.abs(emb1), np.abs(emb2)))
+    union = np.sum(np.maximum(np.abs(emb1), np.abs(emb2)))
+    jaccard = float(intersection / union if union > 0 else 0)
+
+    # Sorensen-Dice Coefficient
+    intersection = np.sum(np.minimum(np.abs(emb1), np.abs(emb2)))
+    sum_magnitudes = np.sum(np.abs(emb1)) + np.sum(np.abs(emb2))
+    sorensen_dice = float(2 * intersection / sum_magnitudes if sum_magnitudes > 0 else 0)
+
+    # Manhattan Distance
+    manhattan = float(np.sum(np.abs(emb1 - emb2)))
+
+    # Pearson Correlation
+    pearson = float(np.corrcoef(emb1, emb2)[0, 1])
+
+    return {
+        'cosine': cosine,
+        'euclidean_distance': euclidean_dist,
+        'euclidean_similarity': euclidean_sim,
+        'jaccard': jaccard,
+        'sorensen_dice': sorensen_dice,
+        'manhattan': manhattan,
+        'pearson': pearson
+    }
+
+def interpret_similarity(score: float, metric: str) -> tuple[str, str]:
+    """
+    Interpret similarity score with emoji and description
+    Returns: (emoji, description)
+    """
+    if metric in ['cosine', 'jaccard', 'sorensen_dice', 'euclidean_similarity']:
+        if score > 0.9:
+            return '🟢', 'Nearly Identical'
+        elif score > 0.7:
+            return '🟢', 'Very Similar'
+        elif score > 0.5:
+            return '🟡', 'Moderately Similar'
+        elif score > 0.3:
+            return '🟠', 'Somewhat Similar'
+        else:
+            return '🔴', 'Different'
+    elif metric == 'pearson':
+        if score > 0.9:
+            return '🟢', 'Strong Positive Correlation'
+        elif score > 0.7:
+            return '🟡', 'Moderate Positive Correlation'
+        elif score > 0.3:
+            return '🟠', 'Weak Positive Correlation'
+        elif score > -0.3:
+            return '⚪', 'No Correlation'
+        elif score > -0.7:
+            return '🟠', 'Weak Negative Correlation'
+        elif score > -0.9:
+            return '🟡', 'Moderate Negative Correlation'
+        else:
+            return '🔴', 'Strong Negative Correlation'
+    else:
+        return '⚪', 'Unknown'
+
+def calculate_similarity(text1: str, text2: str, use_instruction: bool) -> str:
+    """
+    Calculate comprehensive similarity metrics between two texts
+    """
+    if not text1.strip() or not text2.strip():
+        return "❌ Error: Please provide both texts"
+
+    try:
+        embeddings = generate_embeddings([text1, text2], use_instruction)
+
+        # Calculate all similarity metrics
+        emb1 = np.array(embeddings[0])
+        emb2 = np.array(embeddings[1])
+        metrics = calculate_all_similarities(emb1, emb2)
+
+        # Build result string
+        result = "📊 **Comprehensive Similarity Analysis**\n\n"
+
+        # Cosine Similarity (Primary)
+        emoji, interpretation = interpret_similarity(metrics['cosine'], 'cosine')
+        result += f"**Cosine Similarity:** {emoji} {metrics['cosine']:.4f}\n"
+        result += f"└─ {interpretation}\n\n"
+
+        # Jaccard Similarity
+        emoji, interpretation = interpret_similarity(metrics['jaccard'], 'jaccard')
+        result += f"**Jaccard Similarity:** {emoji} {metrics['jaccard']:.4f}\n"
+        result += f"└─ {interpretation}\n\n"
+
+        # Sorensen-Dice Coefficient
+        emoji, interpretation = interpret_similarity(metrics['sorensen_dice'], 'sorensen_dice')
+        result += f"**Sørensen-Dice:** {emoji} {metrics['sorensen_dice']:.4f}\n"
+        result += f"└─ {interpretation}\n\n"
+
+        # Euclidean Distance & Similarity
+        result += f"**Euclidean Distance:** {metrics['euclidean_distance']:.4f}\n"
+        emoji, interpretation = interpret_similarity(metrics['euclidean_similarity'], 'euclidean_similarity')
+        result += f"**Euclidean Similarity:** {emoji} {metrics['euclidean_similarity']:.4f}\n"
+        result += f"└─ {interpretation}\n\n"
+
+        # Manhattan Distance
+        result += f"**Manhattan Distance:** {metrics['manhattan']:.2f}\n\n"
+
+        # Pearson Correlation
+        emoji, interpretation = interpret_similarity(metrics['pearson'], 'pearson')
+        result += f"**Pearson Correlation:** {emoji} {metrics['pearson']:.4f}\n"
+        result += f"└─ {interpretation}\n\n"
+
+        # Overall assessment (based on cosine as primary)
+        result += "---\n**Overall Assessment:**\n"
+        cosine_emoji, cosine_interpretation = interpret_similarity(metrics['cosine'], 'cosine')
+        result += f"{cosine_emoji} {cosine_interpretation} (Cosine: {metrics['cosine']:.4f})"
+
         return result
     except Exception as e:
-        return f"Error: {str(e)}"
+        return f"❌ Error: {str(e)}"
 
 # Create Gradio interface
-with gr.Blocks(title="FairFate Embeddings API") as demo:
+with gr.Blocks(title="FairFate Embeddings API - Qwen3", theme=gr.themes.Soft()) as demo:
     gr.Markdown("""
-    # FairFate Embeddings API
-    
-    Generate semantic embeddings using sentence-transformers/all-MiniLM-L6-v2
-    - **Dimensions:** 384
-    - **Max Tokens:** 256
-    - **Normalized:** Yes (ready for cosine similarity)
-    
-    ## API Usage
-    
-    **Endpoint:** `https://your-username-fairfate-embeddings.hf.space/api/predict`
-    
-    **Request:**
-    ```json
-    {
-      "data": ["Your text here", "Another text"]
-    }
-    ```
-    
-    **Response:**
-    ```json
-    {
-      "data": [[[0.123, -0.456, ...]], [[0.789, -0.012, ...]]]
-    }
-    ```
+    # 🎲 FairFate Embeddings API
+    **Powered by Qwen3-Embedding-0.6B** - #1 Multilingual Embedding Model
+    - 🌍 **100+ Languages** (English, Spanish, French, German, Chinese, Japanese, etc.)
+    - 📐 **1024 Dimensions** (flexible 32-1024)
+    - 📚 **32K Context** (massive text support)
+    - ⚡ **Instruction-Aware** (optimized for RPG content)
+    - 🏆 **#1 on MTEB** Multilingual Leaderboard
+    Perfect for: Product classification, semantic search, recommendations, multilingual matching
     """)
-    
-    with gr.Tab("Test Embeddings"):
-        input_text = gr.Textbox(
-            label="Input Texts (one per line)",
-            placeholder="Enter texts, one per line:\nDungeons and Dragons adventure\nSci-fi space opera campaign",
-            lines=5
+
+    with gr.Tab("🔮 Generate Embeddings"):
+        gr.Markdown("""
+        Generate semantic embeddings for product descriptions, titles, or any text.
+        Enter one text per line for batch processing.
+        """)
+
+        with gr.Row():
+            with gr.Column():
+                input_text = gr.Textbox(
+                    label="Input Texts (one per line)",
+                    placeholder="Example:\nStorm King's Thunder - Epic D&D 5E adventure\nCurse of Strahd - Gothic horror campaign\nPathfinder 2E Core Rulebook",
+                    lines=8
+                )
+                use_inst = gr.Checkbox(label="Use instruction prefix (recommended for RPG content)", value=True)
+                output_dims = gr.Slider(
+                    minimum=32, maximum=1024, value=1024, step=32,
+                    label="Output Dimensions"
+                )
+                submit_btn = gr.Button("Generate Embeddings", variant="primary")
+
+            with gr.Column():
+                output_text = gr.Textbox(label="Results", lines=12)
+
+        submit_btn.click(batch_generate, inputs=[input_text, use_inst, output_dims], outputs=output_text)
+
+        gr.Examples(
+            examples=[
+                ["D&D 5E epic fantasy adventure with dragons and dungeons", True, 1024],
+                ["Cyberpunk shadowrun detective noir campaign\nPathfinder 2E beginner box starter set\nCall of Cthulhu horror investigation", True, 1024],
+            ],
+            inputs=[input_text, use_inst, output_dims],
+        )
+
+    with gr.Tab("🔍 Similarity Calculator"):
+        gr.Markdown("""
+        **Comprehensive Similarity Analysis** - Compare two texts using multiple metrics:
+        - **Cosine Similarity**: Angle between vectors (best for semantic meaning)
+        - **Jaccard Similarity**: Intersection over union (set-like comparison)
+        - **Sørensen-Dice**: Weighted intersection (emphasizes shared features)
+        - **Euclidean Distance/Similarity**: Straight-line distance in vector space
+        - **Manhattan Distance**: Grid-based distance (L1 norm)
+        - **Pearson Correlation**: Linear relationship between vectors
+        Perfect for duplicate detection, classification testing, and understanding product relationships!
+        """)
+
+        with gr.Row():
+            with gr.Column():
+                text1 = gr.Textbox(
+                    label="First Text",
+                    placeholder="Storm King's Thunder - Giant-themed D&D adventure",
+                    lines=3
+                )
+                text2 = gr.Textbox(
+                    label="Second Text",
+                    placeholder="Princes of the Apocalypse - Elemental evil campaign",
+                    lines=3
+                )
+                use_inst_sim = gr.Checkbox(label="Use instruction prefix", value=True)
+                calc_btn = gr.Button("Calculate Similarity", variant="primary")
+
+            with gr.Column():
+                similarity_output = gr.Textbox(label="Similarity Result", lines=8)
+
+        calc_btn.click(calculate_similarity, inputs=[text1, text2, use_inst_sim], outputs=similarity_output)
+
+        gr.Examples(
+            examples=[
+                ["D&D 5E fantasy adventure", "Dungeons and Dragons fifth edition module", True],
+                ["Horror investigation mystery", "Comedy fantasy lighthearted fun", True],
+                ["Pathfinder 2E rulebook", "D&D 5E Player's Handbook", True],
+            ],
+            inputs=[text1, text2, use_inst_sim],
         )
-        output_text = gr.Textbox(label="Results", lines=10)
-        submit_btn = gr.Button("Generate Embeddings")
-        submit_btn.click(batch_generate, inputs=input_text, outputs=output_text)
-    
-    with gr.Tab("API Documentation"):
+
+    with gr.Tab("📖 API Documentation"):
         gr.Markdown("""
-        ### Python Example
+        ## 🚀 Quick Start
+        ### Python
         ```python
         import requests
-        
+        import numpy as np
+        url = "https://YOUR_USERNAME-fairfate-embeddings.hf.space/api/predict"
+        # Generate embeddings
+        texts = [
+            "Storm King's Thunder - Epic D&D 5E adventure",
+            "Curse of Strahd - Gothic horror campaign"
+        ]
         response = requests.post(
-            "https://your-username-fairfate-embeddings.hf.space/api/predict",
-            json={"data": ["Your text here"]}
+            url,
+            json={
+                "data": [texts, True, 1024],  # [texts, use_instruction, dimensions]
+                "fn_index": 0  # Index of generate_embeddings function
+            }
         )
-        embeddings = response.json()["data"][0]
+        result = response.json()
+        embeddings = result["data"][0]
+        print(f"Generated {len(embeddings)} embeddings")
+        print(f"Dimensions: {len(embeddings[0])}")
         ```
-        
-        ### JavaScript/TypeScript Example
+        ### TypeScript/JavaScript
         ```typescript
-        const response = await fetch(
-            'https://your-username-fairfate-embeddings.hf.space/api/predict',
-            {
-                method: 'POST',
-                headers: { 'Content-Type': 'application/json' },
-                body: JSON.stringify({ data: ["Your text here"] })
-            }
-        );
+        const url = 'https://YOUR_USERNAME-fairfate-embeddings.hf.space/api/predict';
+        const response = await fetch(url, {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/json' },
+          body: JSON.stringify({
+            data: [
+              ["Your text here", "Another text"],
+              true,   // use_instruction
+              1024    // output_dimensions
+            ],
+            fn_index: 0
+          })
+        });
         const result = await response.json();
         const embeddings = result.data[0];
         ```
+        ### cURL
+        ```bash
+        curl -X POST \\
+          https://YOUR_USERNAME-fairfate-embeddings.hf.space/api/predict \\
+          -H "Content-Type: application/json" \\
+          -d '{
+            "data": [["Your text here"], true, 1024],
+            "fn_index": 0
+          }'
+        ```
+        ## 📊 Parameters
+        | Parameter | Type | Default | Description |
+        |-----------|------|---------|-------------|
+        | `texts` | string[] | required | Array of texts to embed |
+        | `use_instruction` | boolean | true | Add instruction prefix (improves accuracy) |
+        | `output_dimensions` | number | 1024 | Output size (32-1024) |
+        ## 🎯 Use Cases
+        - **Product Classification**: Auto-tag by genre, system, theme
+        - **Semantic Search**: Find by meaning, not keywords
+        - **Recommendations**: "Similar products"
+        - **Duplicate Detection**: Find similar listings
+        - **Multilingual Matching**: Cross-language similarity
+        ## ⚡ Performance
+        | Batch Size | GPU Throughput | CPU Throughput |
+        |------------|----------------|----------------|
+        | 1          | ~800/sec       | ~80/sec        |
+        | 32         | ~4000/sec      | ~250/sec       |
+        ## 🌍 Supported Languages
+        English, Spanish, French, German, Italian, Portuguese, Russian, Polish, Dutch, Czech,
+        Chinese, Japanese, Korean, Arabic, Hebrew, Hindi, Thai, Vietnamese, Indonesian,
+        Turkish, Swedish, Norwegian, Danish, Finnish, Greek, Romanian, Hungarian, and 80+ more!
+        ## 📝 Citation
+        ```bibtex
+        @misc{qwen3embedding2025,
+          title={Qwen3 Embedding},
+          author={Alibaba Cloud},
+          year={2025},
+          url={https://github.com/QwenLM/Qwen3-Embedding}
+        }
+        ```
         """)
+
+    with gr.Tab("ℹ️ Model Info"):
+        gr.Markdown(f"""
+        ## Model Details
+        - **Model:** {MODEL_NAME}
+        - **Dimensions:** {model.get_sentence_embedding_dimension()}
+        - **Max Sequence Length:** {model.max_seq_length} tokens
+        - **Languages:** 100+
+        - **License:** Apache 2.0
+        - **Normalization:** L2 normalized (ready for cosine similarity)
+        ## Advantages
+        ✅ **Best Multilingual Performance** - #1 on MTEB leaderboard
+        ✅ **Massive Context** - 32K tokens (vs 512 for most models)
+        ✅ **Instruction-Aware** - Can customize for specific domains
+        ✅ **Flexible Dimensions** - 32 to 1024 dimensions
+        ✅ **Code-Switching** - Handles mixed-language text
+        ## Resources
+        - [Model Card](https://huggingface.co/Qwen/Qwen3-Embedding-0.6B)
+        - [GitHub](https://github.com/QwenLM/Qwen3-Embedding)
+        - [Blog Post](https://qwenlm.github.io/blog/qwen3-embedding/)
+        - [MTEB Leaderboard](https://huggingface.co/spaces/mteb/leaderboard)
+        """)
+
 # Launch with API enabled
-demo.launch()
+if __name__ == "__main__":
+    demo.launch()
+