Addressing warnings.

travis-bauer · travis-bauer · commit 79f39876844f · 2025-09-04T05:35:08.000-06:00
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -10,7 +10,7 @@ updates:
     reviewers:
       - "tlbauer"
     assignees:
-      - "tlbauer"
+      - "tlbauer2"
     commit-message:
       prefix: "deps"
       include: "scope"
@@ -40,7 +40,7 @@ updates:
     reviewers:
       - "tlbauer"
     assignees:
-      - "tlbauer"
+      - "tlbauer2"
     commit-message:
       prefix: "docker"
       include: "scope"
diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml
@@ -74,12 +74,12 @@ jobs:
     - name: Run Bandit security scan
       run: |
         bandit -r src/ -f json -o bandit-report.json || true
-        bandit -r src/ -f txt || true  # TODO: Revisit - consider failing build on security issues
+        bandit -r src/ -f txt 
     
     - name: Run Safety dependency scan
       run: |
         safety check --json --output safety-report.json || true
-        safety check || true  # TODO: Revisit - consider failing build on dependency vulnerabilities
+        safety check 
     
     - name: Upload security scan results
       uses: actions/upload-artifact@v4
diff --git a/.safety-project.ini b/.safety-project.ini
@@ -0,0 +1,5 @@
+[project]
+id = talkpipe
+url = /codebases/talkpipe/findings
+name = talkpipe
+
diff --git a/src/talkpipe/app/chatterlang_reference_browser.py b/src/talkpipe/app/chatterlang_reference_browser.py
@@ -352,20 +352,20 @@ def main():
                 print("Attempting to run 'chatterlang_reference_generator' command to generate them...")
                 
                 try:
-                    import subprocess
-                    subprocess.run(['chatterlang_reference_generator'], capture_output=True, text=True, check=True)
+                    from talkpipe.app.chatterlang_reference_generator import go as generate_reference
+                    generate_reference()
                     print("Successfully generated reference files.")
                     
                     # Check if files were created
                     if txt_file.exists():
                         doc_path = str(current_dir)
                     else:
-                        print("Error: talkpipe_ref command completed but files were not created")
+                        print("Error: reference generation completed but files were not created")
                         sys.exit(1)
-                except (subprocess.CalledProcessError, FileNotFoundError) as e:
-                    print(f"Error running talkpipe_ref command: {e}")
+                except Exception as e:
+                    print(f"Error running reference generator: {e}")
                     print("Please either:")
-                    print("  1. Install TalkPipe and ensure 'talkpipe_ref' is in PATH, or")
+                    print("  1. Install TalkPipe properly, or")
                     print("  2. Provide path to directory containing talkpipe_ref.txt")
                     sys.exit(1)
     else:
diff --git a/src/talkpipe/app/chatterlang_serve.py b/src/talkpipe/app/chatterlang_serve.py
@@ -76,13 +76,14 @@ def add_output(self, output: str, message_type: str = "response"):
                 "type": message_type
             }
             self.output_queue.put(timestamped_output, block=False)
-        except:
+        except Exception as e:
             # Queue is full, remove oldest item
+            logger.warning(f"Output queue full, attempting to remove oldest item: {e}")
             try:
                 self.output_queue.get_nowait()
                 self.output_queue.put(timestamped_output, block=False)
-            except:
-                pass
+            except Exception as e2:
+                logger.warning(f"Failed to add output to queue even after removing oldest item: {e2}")
     
     def update_activity(self):
         """Update last activity timestamp"""
@@ -125,7 +126,7 @@ class ChatterlangServer:
     
     def __init__(
         self,
-        host: str = "0.0.0.0",
+        host: str = "localhost",
         port: int = 9999,
         api_key: str = "your-secret-key-here",
         require_auth: bool = False,
@@ -637,11 +638,11 @@ def _get_stream_interface(self) -> str:
         
         form_fields = self._generate_form_fields()
         
-        return f'''
+        template = '''
         <!DOCTYPE html>
         <html>
         <head>
-            <title>{self.title} - Stream</title>
+            <title>{title} - Stream</title>
             <style>
                 * {{
                     margin: 0;
@@ -983,7 +984,7 @@ def _get_stream_interface(self) -> str:
         </head>
         <body>
             <div class="header">
-                <h1>{self.form_config.title}</h1>
+                <h1>{form_config_title}</h1>
             </div>
             
             <div class="main-container">
@@ -1169,7 +1170,7 @@ def _get_stream_interface(self) -> str:
                     }}
                     
                     // Add user message to chat immediately for instant feedback
-                    const displayProperty = '{self.display_property}' || Object.keys(data)[0];
+                    const displayProperty = '{display_property}' || Object.keys(data)[0];
                     const userMessage = data[displayProperty] || JSON.stringify(data);
                     lastUserMessage = userMessage; // Store to detect duplicates from server
                     addMessage(userMessage, 'user', new Date().toISOString());
@@ -1228,6 +1229,34 @@ def _get_stream_interface(self) -> str:
         </body>
         </html>
         '''
+        return template.format(
+            title=self.title,
+            height=height,
+            form_style=form_style,
+            input_bg=input_bg,
+            text_color=text_color,
+            border_color=border_color,
+            button_bg=button_bg,
+            button_hover=button_hover,
+            form_fields=form_fields,
+            auth_header=auth_header,
+            form_config_title=self.form_config.title,
+            bg_color=bg_color,
+            output_bg=output_bg,
+            main_container_style=main_container_style,
+            form_panel_width=form_panel_width,
+            form_panel_style=form_panel_style,
+            chat_panel_style=chat_panel_style,
+            user_msg_bg=user_msg_bg,
+            response_msg_bg=response_msg_bg,
+            output_text=output_text,
+            error_msg_bg=error_msg_bg,
+            controls_style=controls_style,
+            form_panel_class=form_panel_class,
+            chat_controls=chat_controls,
+            standalone_controls=standalone_controls,
+            display_property=self.display_property
+        )
     
     def _get_html_interface(self) -> str:
         """Generate HTML interface with configurable form"""
@@ -1270,11 +1299,11 @@ def _get_html_interface(self) -> str:
         
         form_fields = self._generate_form_fields()
         
-        return f'''
+        template2 = '''
         <!DOCTYPE html>
         <html>
         <head>
-            <title>{self.title}</title>
+            <title>{title}</title>
             <style>
                 * {{
                     margin: 0;
@@ -1483,10 +1512,10 @@ def _get_html_interface(self) -> str:
         <body>
             <div class="main-content">
                 <div class="info-section">
-                    <h1>{self.title}</h1>
+                    <h1>{title}</h1>
                     <p>Submit JSON data using the form below or send POST requests to:</p>
-                    <div class="endpoint-info">POST http://{self.host}:{self.port}/process</div>
-                    {"<p>Authentication required: Include 'X-API-Key' header</p>" if self.require_auth else ""}
+                    <div class="endpoint-info">POST http://{host}:{port}/process</div>
+                    {require_auth_text}
                     <p>View API documentation at: <a href="/docs">/docs</a></p>
                     <p>View streaming interface at: <a href="/stream">/stream</a></p>
                 </div>
@@ -1502,7 +1531,7 @@ def _get_html_interface(self) -> str:
             <div class="form-panel" id="formPanel">
                 <div class="form-container">
                     <div class="form-header">
-                        <h3>{self.form_config.title}</h3>
+                        <h3>{form_config_title}</h3>
                     </div>
                     
                     {auth_header}
@@ -1643,6 +1672,22 @@ def _get_html_interface(self) -> str:
         </body>
         </html>
         '''
+        return template2.format(
+            title=self.title,
+            height=height,
+            form_style=form_style,
+            input_bg=input_bg,
+            text_color=text_color,
+            border_color=border_color,
+            button_bg=button_bg,
+            button_hover=button_hover,
+            form_fields=form_fields,
+            auth_header=auth_header,
+            form_config_title=self.form_config.title,
+            host=self.host,
+            port=self.port,
+            require_auth_text='<p>Authentication required: Include "X-API-Key" header</p>' if self.require_auth else ''
+        )
     
     def set_processor_function(self, func: Callable[[Dict[str, Any]], Any]):
         """Set the function used to process incoming JSON data"""
@@ -1696,7 +1741,7 @@ def load_form_config(config_path: str) -> Dict[str, Any]:
 class ChatterlangServerSegment(AbstractSource):
     """Segment for receiving JSON data via FastAPI with configurable form"""
     
-    def __init__(self, port: Union[int,str] = 9999, host: str = "0.0.0.0", 
+    def __init__(self, port: Union[int,str] = 9999, host: str = "localhost", 
                  api_key: str = None, require_auth: bool = False,
                  form_config: Union[str, Dict[str, Any]] = None):
         super().__init__()
@@ -1776,8 +1821,8 @@ def go():
     parser = argparse.ArgumentParser(description='FastAPI JSON Data Receiver with Configurable Form')
     parser.add_argument('-p', '--port', type=int, default=2025,
                         help='Port to listen on (default: 2025)')
-    parser.add_argument('-o', '--host', default='0.0.0.0',
-                        help='Host to bind to (default: 0.0.0.0)')
+    parser.add_argument('-o', '--host', default='localhost',
+                        help='Host to bind to (default: localhost)')
     parser.add_argument('--api-key', help='Set API key for authentication')
     parser.add_argument('--require-auth', action='store_true',
                         help='Require API key authentication')
diff --git a/src/talkpipe/data/email.py b/src/talkpipe/data/email.py
@@ -173,8 +173,10 @@ def sendEmail(items, subject_field, body_fields, sender_email, recipient_email,
         - Uses TLS encryption for email transmission
     """
     logger.debug(f"Starting sendEmail with subject_field={subject_field}, body_fields={body_fields}")
-    assert subject_field is not None, "subject_field is required"
-    assert body_fields is not None, "body_fields is required"
+    if subject_field is None:
+        raise ValueError("subject_field is required")
+    if body_fields is None:
+        raise ValueError("body_fields is required")
 
     config = get_config()
     logger.debug(f"Loaded config: {config}")
@@ -347,7 +349,8 @@ def fetch_emails(
             
             try:
                 date_obj = email.utils.parsedate_to_datetime(date_str)
-            except:
+            except Exception as e:
+                logger.warning(f"Failed to parse email date '{date_str}': {e}. Using current time.")
                 date_obj = datetime.datetime.now()
             
             # Extract content
diff --git a/src/talkpipe/data/html.py b/src/talkpipe/data/html.py
@@ -98,26 +98,34 @@ def htmlToTextSegment(raw, cleanText=True):
 def get_robot_parser(domain, timeout=5):
     """Retrieve or create a RobotFileParser for a given domain with a timeout."""
     robots_url = f"{domain}/robots.txt"
+    
+    # Validate URL scheme for security
+    parsed_url = urllib.parse.urlparse(robots_url)
+    if parsed_url.scheme not in ('http', 'https'):
+        logger.warning(f"Unsafe URL scheme '{parsed_url.scheme}' in robots_url: {robots_url}. Only http/https allowed.")
+        return None
+    
     rp = RobotFileParser()
     rp.set_url(robots_url)
 
     try:
-        with urllib.request.urlopen(robots_url, timeout=timeout) as response:
-            content = response.read()
-            if content.startswith(b'\x1f\x8b'):
-                # If the content is gzipped, decompress it
-                content = gzip.decompress(content).decode('utf-8')
-            else:
-                # If not gzipped, decode it directly
-                content = content.decode('utf-8')
+        response = requests.get(robots_url, timeout=timeout)
+        response.raise_for_status()  # Raise an exception for bad status codes
+        content_bytes = response.content
+        if content_bytes.startswith(b'\x1f\x8b'):
+            # If the content is gzipped, decompress it
+            content = gzip.decompress(content_bytes).decode('utf-8')
+        else:
+            # If not gzipped, decode it directly
+            content = content_bytes.decode('utf-8')
         
         try:
             rp.parse(content.splitlines())
         except Exception as e:
             # If parsing fails, log the error but return the robot parser anyway
             logger.warning(f"Error parsing robots.txt from {robots_url}: {e}")
             
-    except (urllib.error.URLError, ConnectionError, TimeoutError) as e:
+    except (requests.RequestException, ConnectionError, TimeoutError) as e:
         logger.warning(f"Failed to fetch robots.txt from {robots_url}. Assuming allowed. Error: {e}")
         return None  # Use None to indicate failure to fetch
 
diff --git a/src/talkpipe/operations/filtering.py b/src/talkpipe/operations/filtering.py
@@ -40,10 +40,10 @@ def _hashes(self, item):
         # Convert the item to bytes (if it's not already) so it can be hashed.
         item_bytes = str(item).encode('utf-8')
         
-        # First hash using MD5
-        hash1 = int(hashlib.md5(item_bytes).hexdigest(), 16)
-        # Second hash using SHA-1
-        hash2 = int(hashlib.sha1(item_bytes).hexdigest(), 16)
+        # First hash using MD5 (not for security, used for bloom filter hashing)
+        hash1 = int(hashlib.md5(item_bytes, usedforsecurity=False).hexdigest(), 16)
+        # Second hash using SHA-1 (not for security, used for bloom filter hashing)
+        hash2 = int(hashlib.sha1(item_bytes, usedforsecurity=False).hexdigest(), 16)
         
         # Generate hash values using double hashing:
         # For each i, compute: (hash1 + i * hash2) mod size
diff --git a/src/talkpipe/operations/transforms.py b/src/talkpipe/operations/transforms.py
@@ -84,7 +84,8 @@ def fill_null(items, default='', **kwargs):
         [{'a': None, 'b': 1}, {'a': 2, 'b': 'EMPTY'}]
     """
     for item in items:
-        assert isinstance(item, dict), f"Expected a dictionary, but got {type(item)} instead"
+        if not isinstance(item, dict):
+            raise TypeError(f"Expected a dictionary, but got {type(item)} instead")
         for k in item:
             if item[k] is None and k not in kwargs:
                 item[k] = default
@@ -129,7 +130,8 @@ def transform(self, input_iter):
 
         """
 
-        assert self.num_items is None or self.num_items > 0, "num_vectors must be None or a positive integer"
+        if self.num_items is not None and self.num_items <= 0:
+            raise ValueError("num_vectors must be None or a positive integer")
         accumulated = []
         for datum in input_iter:
             item = extract_property(datum, self.field)
diff --git a/src/talkpipe/pipe/basic.py b/src/talkpipe/pipe/basic.py
@@ -555,7 +555,8 @@ def fillTemplate(item, template:str, fail_on_missing:bool=True, default: Optiona
     Returns:
         str: The filled template string
     """
-    assert template is not None
+    if template is None:
+        raise ValueError("Template cannot be None")
     field_names = extract_template_field_names(template)
     values = {field_name:extract_property(item, field_name, fail_on_missing=fail_on_missing, default=default) for field_name in field_names}
     return fill_template(template, values)
diff --git a/src/talkpipe/util/collections.py b/src/talkpipe/util/collections.py
@@ -117,6 +117,7 @@ def _load(self):
                 self.data = db.get('data', {})
                 self.expiry = db.get('expiry', {})
             self._clean_expired()
-        except:
+        except Exception as e:
+            logger.warning(f"Failed to load cached data from {self.filename}: {e}. Starting with empty cache.")
             self.data = {}
             self.expiry = {}
diff --git a/src/talkpipe/util/data_manipulation.py b/src/talkpipe/util/data_manipulation.py
@@ -342,8 +342,14 @@ def lambda_function(item: Any) -> Any:
         restricted_globals.update(SAFE_BUILTINS)
 
         # Evaluate the expression in a heavily restricted environment
+        # Note: eval() is used intentionally here with extensive security controls:
+        # - Restricted globals (no dangerous builtins)
+        # - Input validation (blocks dangerous patterns)
+        # - Compiled code with syntax checking
+        # - Exception handling for safety
+        # ast.literal_eval() cannot be used as this evaluates dynamic expressions, not just literals
         try:
-            result = eval(compiled_code, restricted_globals, locals_dict)
+            result = eval(compiled_code, restricted_globals, locals_dict)  # nosec B307
             return result
         except Exception as e:
             error_msg = f"Error evaluating expression '{expression}' on item {item}: {e}"
diff --git a/tests/talkpipe/app/test_chatterlang_serve.py b/tests/talkpipe/app/test_chatterlang_serve.py
@@ -226,7 +226,7 @@ def test_init_default_config(self):
         """Test server initialization with default configuration."""
         server = ChatterlangServer()
         
-        assert server.host == "0.0.0.0"
+        assert server.host == "localhost"
         assert server.port == 9999
         assert server.require_auth is False
         assert server.title == "ChatterLang Server"
@@ -823,7 +823,7 @@ def test_go_default_args(self, mock_server_class):
             # Verify server was created with default values
             mock_server_class.assert_called_once()
             call_args = mock_server_class.call_args[1]
-            assert call_args["host"] == "0.0.0.0"
+            assert call_args["host"] == "localhost"
             assert call_args["port"] == 2025
             assert call_args["require_auth"] is False
             
diff --git a/tests/talkpipe/search/test_whoosh.py b/tests/talkpipe/search/test_whoosh.py
