Merge pull request llmware-ai#701 from llmware-ai/remove-werkzeug-utils-dependency

removing werkzeug.utils dependency

Author: doberst

llmware/library.py

@@ -20,7 +20,6 @@
 updating, and other tasks pertaining to Libraries via the Library Card.
 """
 
-from werkzeug.utils import secure_filename
 import shutil
 import os
 import json
@@ -141,7 +140,7 @@ def create_new_library(self, library_name, account_name="llmware"):
         self.account_name = account_name
 
         # apply safety check to library_name path
-        library_name = secure_filename(library_name)
+        library_name = Utilities().secure_filename(library_name)
 
         library_exists = self.check_if_library_exists(library_name,account_name)
 
@@ -150,6 +149,9 @@ def create_new_library(self, library_name, account_name="llmware"):
             logging.info("update: library already exists - returning library - %s - %s ", library_name, account_name)
             return self.load_library(library_name, account_name)
 
+        # assign self.library_name to the 'safe' library_name
+        self.library_name = library_name
+
         # allow 'dynamic' creation of a new account path
         account_path = os.path.join(LLMWareConfig.get_library_path(), account_name)
         if not os.path.exists(account_path):

llmware/parsers.py

@@ -39,7 +39,6 @@
 
 import time
 import json
-from werkzeug.utils import secure_filename
 import os
 from zipfile import ZipFile, ZIP_DEFLATED
 import shutil
@@ -555,7 +554,7 @@ def zip_extract_handler(self):
             if success_code == 1:
 
                 #   iterate thru all of the files found in the zip archive
-                #   apply secure_filename and prep_filename
+                #   apply secure filename and prep filename
                 #   route to the appropriate work folder, if applicable
 
                 for f in z.namelist():
@@ -2163,7 +2162,8 @@ def parse_pdf_by_ocr_images(self, input_fp, write_to_db=True, save_history=True,
 
                 ext = file.split(".")[-1]
                 if ext == "pdf":
-                    doc_fn = secure_filename(file)
+
+                    doc_fn = Utilities().secure_filename(file)
 
                     # get new doc_ID number
                     if write_to_db_on == 1:
@@ -2417,7 +2417,7 @@ def parse_wiki(self, topic_list, write_to_db=True, save_history=False, target_re
 
         for i, topic in enumerate(topic_list):
 
-            fn = "wiki-topic-" + secure_filename(topic) + ".txt"
+            fn = "wiki-topic-" + Utilities().secure_filename(topic) + ".txt"
 
             logging.info("update: parse_wiki - %s - %s", topic, fn)
 
@@ -2961,8 +2961,8 @@ def parse_website(self, url_base, write_to_db=True, save_history=True, get_links
 
         website_name = "my_website.html"
 
-        # apply secure_filename to remove any extra "/"
-        secure_url_name = secure_filename(website.url_main.split(".")[-2])
+        # apply secure filename to remove any extra "/"
+        secure_url_name = Utilities().secure_filename(website.url_main.split(".")[-2])
 
         out_name = str(random.randint(100000, 999999)) + "_" + secure_url_name + ".html"
 
@@ -3013,7 +3013,7 @@ def prep_filename(self, fn, secure_name=True, prepend_string=None, postpend_stri
 
         # default - apply basic secure name, e.g., remove / and insert _
         if secure_name:
-            fn_out= secure_filename(fn)
+            fn_out= Utilities().secure_filename(fn)
 
         # if requested prepend or postpend
         if prepend_string:
@@ -3094,7 +3094,7 @@ def input_ingestion_comparison_from_parser_state (self, file_list):
             found_file = -1
             for j, ingested_file in enumerate(doc_fn_out):
 
-                # need to confirm 'symmetrical' transformations, e.g., secure_filename and any prepend/postpend
+                # need to confirm 'symmetrical' transformations, e.g., secure filename and any prepend/postpend
                 if input_file == ingested_file:
                     found_file = 1
                     found_list.append(input_file)
@@ -3357,7 +3357,7 @@ def parse_one_pdf_by_ocr_images(self, input_fp, input_fn, save_history=True):
 
         if ext == "pdf":
 
-            doc_fn = secure_filename(input_fn)
+            doc_fn = Utilities().secure_filename(input_fn)
 
             output_by_page = ImageParser(self).process_pdf_by_ocr(input_fp, input_fn)
 
@@ -3403,7 +3403,7 @@ def parse_one_image(self, input_fp, input_fn, save_history=True):
 
         if ext in self.ocr_types:
 
-            doc_fn = secure_filename(input_fn)
+            doc_fn = Utilities().secure_filename(input_fn)
             ocr_output = ImageParser(self).process_ocr(input_fp, input_fn)
 
             meta = {"author": "", "modified_date": "", "created_date": "", "creator_tool": ""}

llmware/requirements.txt

@@ -14,7 +14,6 @@ tabulate==0.9.0
 tokenizers>=0.15.0
 torch>=1.13.1
 transformers>=4.36.0
-Werkzeug==3.0.1
 word2number==1.1
 Wikipedia-API==0.6.0
 yfinance>=0.2.28

llmware/resources.py

@@ -45,8 +45,6 @@
 except ImportError:
     pass
 
-from werkzeug.utils import secure_filename
-
 from llmware.configs import LLMWareConfig, PostgresConfig, LLMWareTableSchema, SQLiteConfig, AWSS3Config
 
 from llmware.exceptions import LLMWareException, UnsupportedCollectionDatabaseException, InvalidNameException
@@ -4248,7 +4246,15 @@ def connect_to_user_s3_bucket (self, aws_access_key, aws_secret_key,
             files = bucket.objects.all()
 
             for file in files:
-                f = secure_filename(file.key)
+
+                # strip os.sep from file name
+                safe_file_name = str(file.key)
+                if safe_file_name.startswith(os.sep):
+                    safe_file_name = safe_file_name[1:]
+
+                f = safe_file_name.replace(os.sep, "_")
+                f = f.replace(" ", "_")
+
                 file_type = f.split(".")[-1].lower()
                 if file_type in accepted_file_formats:
                     s3.download_file(user_bucket_name, file.key, local_download_path + f)

llmware/util.py

@@ -833,6 +833,23 @@ def convert_media_file_to_wav(self, path_to_file_to_convert, save_path=None, fil
 
         return outfile_path
 
+    def secure_filename(self, fn):
+
+        """ New utility method to remove os.sep from proposed filenames. """
+
+        # strip os.sep from file name
+        safe_file_name = str(fn)
+        if safe_file_name.startswith(os.sep):
+            safe_file_name = safe_file_name[1:]
+
+        # removes os separator
+        secure_fn = safe_file_name.replace(os.sep, "_")
+
+        # converts spaces into underscores
+        secure_fn = secure_fn.replace(" ", "_")
+
+        return secure_fn
+
 
 class CorpTokenizer:
 

setup.py

@@ -44,8 +44,10 @@ def glob_fix(package_name, glob):
         "License :: OSI Approved :: Apache Software License",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12"
     ],
-    keywords="ai,data,development",  # Optional 
+    keywords="ai,llm,rag,data,development",  # Optional
     packages=['llmware'],
     package_data={'llmware': ['*.c', '*.so', '*.dylib', '.dylibs/*', *glob_fix('llmware', 'lib/**/*')], 'llmware.libs': ['*']},
     python_requires=">=3.9",
@@ -67,7 +69,6 @@ def glob_fix(package_name, glob):
         'tokenizers>=0.15.0',
         'torch>=1.13.1',
         'transformers>=4.36.0',
-        'Werkzeug==3.0.1',
         'word2number==1.1',
         'Wikipedia-API==0.6.0',
         'yfinance>=0.2.28',