Merge pull request #730 from sassoftware/opensearch-path-based-fix

Opensearch path based fix

Author: ceelias

logging/bin/getlogs.py

@@ -1,4 +1,4 @@
-#! /usr/bin/python3 
+#! /usr/bin/python3
 
 # Copyright © 2023, SAS Institute Inc., Cary, NC, USA.  All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
@@ -13,7 +13,7 @@
 from subprocess import run
 ##v 0.2.0
 
-def validate_input(checkInput): 
+def validate_input(checkInput):
     """ Validate the arguments passed by the user to ensure script will function properly"""
 
     ##Set maximum log limit for output
@@ -46,20 +46,20 @@ def validate_input(checkInput):
             if (checkInput['force'] == False):
                 print("\nUser specified output file already exists. Use -f to overwrite the file.\n")
                 sys.exit()
-        
+
         safe_dir = os.getcwd() ## Check for path traversal attack
         if os.path.commonprefix((os.path.realpath(checkInput['out-filename']),safe_dir)) != safe_dir:
             print("Error: Out-file path must be in the current working directory.")
             sys.exit()
 
         try:
-            x = open(checkInput['out-filename'], 'w')   
-            x.close() 
+            x = open(checkInput['out-filename'], 'w')
+            x.close()
         except FileNotFoundError as e:
             print("Error: Output file path not found. Please verify output file path. ")
             sys.exit()
 
-    if checkInput['savequery']: ##Find saved query path location     
+    if checkInput['savequery']: ##Find saved query path location
         if(type(checkInput['savequery']) == list):
             checkInput['savequery']= " ".join(checkInput['savequery'])
         if (checkInput['savequery'].find('.') == -1):
@@ -77,8 +77,8 @@ def validate_input(checkInput):
         if (not os.path.isfile(checkInput['query-filename'])):
             print("Error: Invalid query file path.")
             sys.exit()
- 
-    ##Time Validator - Verifies input, and converts it to UTC   
+
+    ##Time Validator - Verifies input, and converts it to UTC
     if (type(checkInput['dateTimeStart']) ==list):
         checkInput['dateTimeStart'] = " ".join(checkInput['dateTimeStart'])
     if (type(checkInput['dateTimeEnd']) == list):
@@ -93,7 +93,7 @@ def validate_input(checkInput):
             checkInput['dateTimeStart'] = time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(time.mktime(checkInput['dateTimeStart'])))
             checkInput['dateTimeEnd'] = time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(time.mktime(checkInput['dateTimeEnd'])))
     except ValueError:
-        print("One or more date(s) have been formatted incorrectly. Correct format is Y-M-D H:M:S. Ex: 1999-02-16 10:00:00")  
+        print("One or more date(s) have been formatted incorrectly. Correct format is Y-M-D H:M:S. Ex: 1999-02-16 10:00:00")
         sys.exit()
 
     if (checkInput['message']): ## Argument formatting for query builder
@@ -125,16 +125,16 @@ def open_port():
     time.sleep(5)
 
     return port
-    
-def build_query(args): 
+
+def build_query(args):
     """Generates Query using Opensearch DSL"""
     """Takes arguments from user and returns a JSON-format query to pass to OpenSearch API"""
-    first = True 
+    first = True
     sourcerequested = False
     argcounter=0    ##Counts unique options entered by user, sets min_match to this number
-    if (not args['query-filename']):  
+    if (not args['query-filename']):
         tfile = tempfile.NamedTemporaryFile(delete = False)  ##If User has not specified query file, create temp file for one.
-        temp = open(tfile.name, 'w') 
+        temp = open(tfile.name, 'w')
 
         temp.write('{"size": ' + str(args['maxInt']) + ',"sort": [{"@timestamp": {"order": "desc","unmapped_type": "boolean"} }]')     ## Establish size of query, remove scoring
         temp.write(', "query": {"bool": {"must":[ {"range": {"@timestamp": {"gte": "' + args['dateTimeStart'] + '","lt": "' + args['dateTimeEnd'] + '"} } }], ')   ##Establish Query with Time Range Requirements
@@ -143,9 +143,9 @@ def build_query(args):
             if (("kube" in argname or "level" in argname or "mes" in argname or "log" in argname) and (args[argname]) and argname.find('-ex')==-1): ##Looking for non-exclusion options that are not NoneType in args argsionary
                 argcounter+=1
                 if (first == True):
-                    first = False  
+                    first = False
                 else:
-                    temp.write(',')       
+                    temp.write(',')
                 if argname!='message':
                     for i in range(len(args[argname])):
                         temp.write('{"match_phrase": { "' + argname + '":"' + args[argname][i] + '" } }')
@@ -183,30 +183,30 @@ def build_query(args):
         else:
            temp.write('"_source": false }')
         temp.close()
-       
+
         temp = open(tfile.name, 'r')
-        query =  " ".join([line.strip() for line in temp]) ## Turns query into string 
+        query =  " ".join([line.strip() for line in temp]) ## Turns query into string
         temp.close()
         tfile.close()
 
     else: ##Open existing query
         x = open(args['query-filename'], 'rt')
         query =  " ".join([line.strip() for line in x]) ## Turn file into string, return.
         x.close()
-    
+
     return query
 
 def get_arguments():
     """List of valid arguments that are read from user as soon as program is run, nargs=+ indicates that argument takes multiple whitespace separated values. """
-   
+
     parser = argparse.ArgumentParser(prog='getLogs.py', usage='\n%(prog)s [options]', description="""This program generates OpenSearch DSL Queries from user specified parameters, and submits them to a database to retrieve logs. The flags below provide specifications for your Query, and can be placed in any order. \n
     \033[1m NOTES: *All default values for username, password, host, and port, are derived from the ENV variables OSUSER, OSPASSWD, OSHOST, OSPORT in that respective order. '\033[0m' \n
     \033[1m If you have default connections set in your environment variables, you can call this program without arguments and get the latest 10 logs from the target API in the default CSV format.  \033[0m \n
     Getlogs has a default set of fields that runs with every query (seen below). You can replace the default fields with your own space-separated set of fields using --fields. Ex: --fields kube.labels.sas_com/deployment properties.appname \n
-    *The NAMESPACE*, POD*, CONTAINER*, LOGSOURCE* and LEVEL* options accept multiple, space-separated, values (e.g. --level INFO NONE). Please refrain from passing single quotes ('') into arguments. \n  
-    *All Generated files are placed in the current working directory. 
+    *The NAMESPACE*, POD*, CONTAINER*, LOGSOURCE* and LEVEL* options accept multiple, space-separated, values (e.g. --level INFO NONE). Please refrain from passing single quotes ('') into arguments. \n
+    *All Generated files are placed in the current working directory.
     Don't include https:\\ in the HOST connection setting \n\n\n \t\t\t\t QUERY SEARCH PARAMETERS: \n\n""", formatter_class=argparse.RawTextHelpFormatter)
-    
+
     ##Search Params
     parser.add_argument('-n', '--namespace', required=False, dest="kube.namespace", nargs='*', metavar="NAMESPACE", help="\nOne or more Viya deployments/Kubernetes Namespace for which logs are sought\n\n")
     parser.add_argument('-nx', '--namespace-exclude', required=False, dest="kube.namespace-ex", nargs='*', metavar="NAMESPACE", help='\nOne or more namespaces for which logs should be excluded from the output\n\n')
@@ -231,7 +231,7 @@ def get_arguments():
     parser.add_argument('-fi','--fields',  required=False, dest="fields", nargs="*", metavar= "FIELDS", default=['@timestamp', 'level', 'kube.pod', 'message'], help = "\n Specify desired output columns from query. If a matching log is returned that does not have the specified field, a NULL value will be used as a placeholder. The _id field is always provided for every log message. \n Default fields: @timestamp level kube.pod message _id\n\n")
     parser.add_argument('-st', '--start', required=False, dest="dateTimeStart", nargs='*', metavar="DATETIME",  default = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.mktime(time.localtime()) - 3600)), help = "\nDatetime for start of period for which logs are sought (default: 1 hour ago). Correct format is Y-M-D H:M:S. Ex: 2023-02-16 10:00:00\n\n")
     parser.add_argument('-en', '--end', required=False, dest="dateTimeEnd",nargs='*', metavar="DATETIME",  default = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), help = "\nDatetime for end of period for which logs are sought (default: now). \n\n\n \t\t\t CONNECTION SETTINGS: \n\n")
-    
+
     parser.add_argument('-i', '--index', required=False, dest="index", metavar="INDEX", default="viya_logs-*") ## help = "\nDetermine which index to perform the search in. Default: viya-logs-*\n\n
     ##Connection settings
     parser.add_argument('-pf','--port-forward', required=False, dest="portforward", action = 'store_true', help = "\n If this option is provided, getlogs will use the value in your KUBECONFIG (case-sensitive) environment variable to port-forward and connect to the OpenSearch API. This skips OSHOST and OSPORT, but OSUSER and OSPASSWD are stil required to authenticate and connect to the database. \n\n")
@@ -240,21 +240,45 @@ def get_arguments():
     parser.add_argument('-ho', '--host', required=False,  dest="host", default=os.environ.get("OSHOST"), help = "\nHostname for connection to OpenSearch Please ensure that host does not contain 'https://' (default: $OSHOST)\n\n")
     parser.add_argument('-po', '--port', required=False,  dest="port", default=os.environ.get("OSPORT"), help = "\nPort number for connection to OpenSearch (default: $OSPORT)\n\n")
     parser.add_argument('-nossl', '--disable-ssl', required=False, dest = "ssl", action= "store_false", help = "\n If this option is provided, SSL will not be used to connect to the database.\n\n")
+
+    # Add arguments for path-based ingress configuration
+    parser.add_argument('--path-based', required=False, dest="path_based", nargs='?',
+                        const="opensearch", default=None,
+                        help="Specify if path-based ingress is used. Without a value, defaults to 'opensearch' prefix. "
+                             "You can also specify a custom prefix, e.g. --path-based my-custom-prefix")
+
     return parser.parse_args().__dict__
 
 def main():
-
+    url_prefix = None
     args = get_arguments() ##Creates "args" dictionary that contains all user submitted options. Print "args" to debug values. Note that the 'dest' value for each argument in argparser object is its key.
     validate_input(args) ##Pass args dictionary for input validation
 
     if args['portforward']: ##Modify connection settings if port forward is selected
         args['host'] = 'localhost'
         args['port'] = open_port()
 
+    if args['path_based']:
+        # User specified path-based ingress - either default or custom
+        url_prefix = args['path_based']  # Will be "opensearch" or custom value
+
     # Establish Client Using User Authorization and Connection Settings
+
+    # Create hosts configuration
+    host_config = {
+        'host': args['host'],
+        'port': args['port']
+    }
+    if url_prefix:
+        host_config['url_prefix'] = url_prefix
+        print(f"Using path-based ingress with prefix: {url_prefix}")
+    else:
+        print("Using host-based ingress (no URL prefix)")
+
     auth = (args['userName'], args['password'])
     client = OpenSearch(
-        hosts = [{'host': args['host'], 'port': args['port']}],
+        # hosts = [{'host': args['host'], 'url_prefix': 'opensearch','port': args['port']}],
+        hosts = [host_config],
         http_compress = True, # enables gzip compression for request bodies
         http_auth = auth,
         # client_cert = client_cert_path,
@@ -270,7 +294,7 @@ def main():
     x = build_query(args)
     index_name = args['index']
 
-    if (args['showquery'] == True): ##Print Query if user asks. 
+    if (args['showquery'] == True): ##Print Query if user asks.
         print("The following query will be submitted:\n\n", json.dumps(json.loads(x), indent=2))
 
     if(args['savequery']): ##Save Query if user asks.
@@ -284,11 +308,11 @@ def main():
             print("Error: Saved query must be written to current working directory.")
             sys.exit()
         print("\nQuery saved to " + args['savequery'])
-        
+
     print('\nSearching index: ')
     try:
         response = client.search(body=x, index=index_name)
-    except Exception as e: 
+    except Exception as e:
         print(e)
         if ("getaddrinfo" in str(e)):
             print("Connection Failed. Please verify the host and port values. ")
@@ -298,7 +322,7 @@ def main():
             print("Username:", args['userName'], " Password:", args['password'], " Host:", args['host'], " Port:", args['port'])
         else:
             print("Connection error. Please verify connection values. ")
-            print("Username:", args['userName'], " Password:", args['password'], " Host:", args['host'], " Port:", args['port']) 
+            print("Username:", args['userName'], " Password:", args['password'], " Host:", args['host'], " Port:", args['port'])
         sys.exit()
 
     if response['hits']['total']['value'] == 0:
@@ -352,7 +376,7 @@ def main():
                 writer.writeheader()
                 for fieldDict in hitsList:
                     writer.writerow(fieldDict)
-                print("Search complete. Results printed to " + args['out-filename'])   
+                print("Search complete. Results printed to " + args['out-filename'])
         else:
             print("Search complete")
             with sys.stdout as csvfile:
@@ -365,4 +389,3 @@ def main():
 
 if __name__ == "__main__":
     main()
-