shifting several web-related dependencies to optional

Author: DARREN OBERST

examples/Parsing/parse_web_sources_in_memory.py

@@ -4,6 +4,13 @@
     When parsing websites, please follow best practices, ethical guidelines and common sense - as a good example,
     see https://monashdatafluency.github.io/python-web-scraping/section-5-legal-and-ethical-considerations/
 
+    To use the WebSite Parser requires several additional python libraries to be installed:
+
+        pip3 install beautifulsoup4
+        pip3 install lxml
+        pip3 install requests
+        pip3 install urllib3
+
 """
 
 from llmware.parsers import Parser, WebSiteParser

examples/Use_Cases/web_services_slim_fx.py

@@ -14,13 +14,20 @@
     The example shows how to extract keys from one source that can then be used as a lookup in a web service to
     supplement the original source materials, and provide a secondary source, which can then also be prompted and
     used to extract, analyze and summarize key information.
+
+    NOTE: to run this example, please install yfinance library, e.g., 'pip3 install yfinance'
+
     """
 
 
 from llmware.web_services import YFinance
 from llmware.models import ModelCatalog
 from llmware.parsers import WikiParser
 
+from importlib import util
+if not util.find_spec("yfinance"):
+    print("\nto run this example, you need to install yfinance first, e.g., pip3 install yfinance")
+
 # our input - financial news article
 
 text=("BEAVERTON, Ore.--(BUSINESS WIRE)--NIKE, Inc. (NYSE:NKE) today reported fiscal 2024 financial results for its "

llmware/requirements.txt

@@ -1,8 +1,6 @@
-beautifulsoup4==4.11.1  
 boto3==1.24.53  
 datasets==2.15.0   
 huggingface-hub==0.19.4  
-lxml==4.9.3  
 numpy>=1.23.2
 openai>=1.0
 pdf2image==1.16.0
@@ -16,7 +14,6 @@ torch>=1.13.1
 transformers>=4.36.0
 word2number==1.1
 Wikipedia-API==0.6.0
-yfinance>=0.2.28
 psycopg-binary==3.1.17
 psycopg==3.1.17
 pgvector==0.2.4  

llmware/web_services.py

@@ -164,8 +164,9 @@ def __init__(self, ticker=None):
 
         try:
             import yfinance
-        except ImportError:
-            raise LLMWareException(message="Exception: need to `pip install yfinance` library.")
+        except ImportError or ModuleNotFoundError:
+            raise LLMWareException(message="Exception: YFinance library not installed - "
+                                           "fix with `pip3 install yfinance`")
 
         if ticker:
             self.company_info = yfinance.Ticker(ticker)
@@ -255,10 +256,15 @@ def __init__(self, url_or_fp, link="/", save_images=True, reset_img_folder=False
             from bs4 import BeautifulSoup
             import requests
             from urllib.request import urlopen, Request
-        except ImportError:
-            raise LLMWareException(message="Exception: to use WebSiteParser requires three additional Python "
-                                           "dependencies via pip install:  bs4 (BeautifulSoup), requests, and "
-                                           "urllib.request")
+            import lxml
+
+        except ModuleNotFoundError or ImportError:
+            raise LLMWareException(message="Exception: to use WebSiteParser requires additional Python "
+                                           "dependencies via pip install:  "
+                                           "\n -- pip3 install beautifulsoup4 (or bs4)"
+                                           "\n -- pip3 install lxml"
+                                           "\n -- pip3 install requests"
+                                           "\n -- pip3 install urllib3")
 
         #   note: for webscraping, unverified ssl are a common error
         #   to debug, if the risk environment is relatively low, set `unverified_context` = True, although
@@ -341,7 +347,7 @@ def __init__(self, url_or_fp, link="/", save_images=True, reset_img_folder=False
             except Exception as e:
                 success_code = -1
                 raise LLMWareException(message=f"Exception: website_parser could not connect to website - "
-                                               f"caught error - {e}.  Two suggested fixes: \n"
+                                               f"caught error - {e}.  Common issues: \n"
                                                f"1.  Update your certificates in the Python path, e.g., "
                                                f"'Install Certificates.command'\n"
                                                f"2.  Set unverified_context=True in the constructor for "

setup.py

@@ -53,11 +53,9 @@ def glob_fix(package_name, glob):
     python_requires=">=3.9",
     zip_safe=True,
     install_requires=[
-        'beautifulsoup4==4.11.1',
         'boto3==1.24.53',
         'datasets==2.15.0',
         'huggingface-hub==0.19.4',
-        'lxml==4.9.3',
         'numpy>=1.23.2',
         'openai>=1.0.0',
         'pdf2image==1.16.0',
@@ -71,7 +69,6 @@ def glob_fix(package_name, glob):
         'transformers>=4.36.0',
         'word2number==1.1',
         'Wikipedia-API==0.6.0',
-        'yfinance>=0.2.28',
         'psycopg-binary==3.1.17',
         'psycopg==3.1.17',
         'pgvector==0.2.4',