updating sentence transformer test example configs

Author: DARREN OBERST

examples/Embedding/using_sentence_transformer.py

@@ -1,7 +1,11 @@
 
-"""This example shows how to use sentence transformers as a vector embedding model with llmware"""
+"""This example shows how to use sentence transformers as a vector embedding model with llmware.
 
-"""Note: this example illustrates capability from llmware==0.1.13 - please update pip install, or pull from repo"""
+To use models from the SentenceTransformer catalog, you may need to install as follows:
+
+    pip3 install sentence-transformers
+
+"""
 
 
 import os
@@ -10,7 +14,12 @@
 from llmware.library import Library
 from llmware.retrieval import Query
 from llmware.models import ModelCatalog
+from llmware.configs import LLMWareConfig
 
+from importlib import util
+if not util.find_spec("sentence_transformers"):
+    print("\nto run this example, you should install the SentenceTransformer library with:  "
+          "pip3 install sentence-transformers.")
 
 def build_lib (library_name, folder="Agreements"):
 
@@ -39,7 +48,9 @@ def build_lib (library_name, folder="Agreements"):
 
 print("update: Step 1- starting here- building library- parsing PDFs into text chunks")
 
-lib = build_lib("st_embedding_0_454")
+LLMWareConfig().set_active_db("sqlite")
+
+lib = build_lib("st_embedding_0")
 
 #   register a model from the sentence transformers library/repository
 

llmware/requirements.txt

@@ -4,7 +4,6 @@ huggingface-hub==0.19.4
 numpy>=1.23.2
 openai>=1.0
 pymongo>=4.7.0
-sentence-transformers==2.2.2
 tabulate==0.9.0
 tokenizers>=0.15.0
 torch>=1.13.1
@@ -18,7 +17,7 @@ colorama==0.4.6
 einops==0.7.0
 librosa>=0.10.0
 
-requests~=2.31.0
+requests>=2.31.0
 tqdm~=4.66.1
 botocore~=1.27.96
 setuptools~=68.2.0

setup.py

@@ -59,7 +59,6 @@ def glob_fix(package_name, glob):
         'numpy>=1.23.2',
         'openai>=1.0.0',
         'pymongo>=4.7.0',
-        'sentence-transformers==2.2.2',
         'tabulate==0.9.0',
         'tokenizers>=0.15.0',
         'torch>=1.13.1',

tests/embeddings/test_all_sentence_transformer_models.py

@@ -0,0 +1,77 @@
+
+""" Tests that sentence transformer model is loaded and yielding a structurally correct embedding vector.
+
+To use this test, you may need install the SentenceTransformer library as follows:
+
+    -- pip3 install sentence-transformers
+
+"""
+
+
+from llmware.models import ModelCatalog
+from sentence_transformers import SentenceTransformer
+
+
+def test_sentence_transformer_model_local_load():
+
+    # This model list was generated by here https://www.sbert.net/docs/pretrained_models.html and
+    # selecting the "All Models" switch
+
+    sentence_transformer_models = [
+        'all-MiniLM-L12-v1',
+        'all-MiniLM-L12-v2',
+        'all-MiniLM-L6-v1',
+        'all-MiniLM-L6-v2',
+        'all-distilroberta-v1',
+        'all-mpnet-base-v1',
+        'all-mpnet-base-v2',
+        'all-roberta-large-v1',
+        'average_word_embeddings_glove.6B.300d',
+        'average_word_embeddings_komninos',
+        'gtr-t5-base',
+        'gtr-t5-large',
+        'gtr-t5-xl',
+        'gtr-t5-xxl',
+        'msmarco-bert-base-dot-v5',
+        'msmarco-distilbert-base-tas-b',
+        'msmarco-distilbert-dot-v5',
+        'multi-qa-MiniLM-L6-cos-v1',
+        'multi-qa-MiniLM-L6-dot-v1',
+        'multi-qa-distilbert-cos-v1',
+        'multi-qa-distilbert-dot-v1',
+        'multi-qa-mpnet-base-cos-v1',
+        'multi-qa-mpnet-base-dot-v1',
+        'paraphrase-MiniLM-L12-v2',
+        'paraphrase-MiniLM-L3-v2',
+        'paraphrase-MiniLM-L6-v2',
+        'paraphrase-TinyBERT-L6-v2',
+        'paraphrase-albert-small-v2',
+        'paraphrase-distilroberta-base-v2',
+        'paraphrase-mpnet-base-v2',
+        'paraphrase-multilingual-MiniLM-L12-v2',
+        'paraphrase-multilingual-mpnet-base-v2',
+        'sentence-t5-base',
+        'sentence-t5-large',
+        'sentence-t5-xl',
+        'sentence-t5-xxl'
+    ]
+
+    test_text = ("This is just a sample text to confirm that the embedding model is loading and correctly "
+                 "converting into a structurally accurate embedding vector.")
+
+    for model_name in sentence_transformer_models:
+
+        print(f"\nloading sentence transformer model: {model_name}")
+
+        st_model = SentenceTransformer(model_name)
+        model = ModelCatalog().load_sentence_transformer_model(st_model, model_name=model_name)
+        embedding_vector = model.embedding([test_text])
+
+        assert embedding_vector is not None
+
+        print(f"created vector successfully with dimensions: ", embedding_vector.shape)
+
+    return 0
+
+
+test_sentence_transformer_model_local_load()