From 235c0e13bb9a8863685dc8c7aa2b67673cac18da Mon Sep 17 00:00:00 2001 From: Pratik Prajapati Date: Mon, 30 Sep 2019 12:16:54 +0530 Subject: [PATCH 1/3] added logging --- metering/handlers/freecall_handler.py | 58 ++++++++++++-------------- metering/handlers/usage_handler.py | 40 ++++++++---------- metering/logger.py | 21 ++++++---- metering/repository/base_repository.py | 2 +- metering/serverless.yml | 6 +-- metering/services.py | 10 ++++- metering/storage.py | 9 +++- metering/utils.py | 35 +++++++--------- 8 files changed, 92 insertions(+), 89 deletions(-) diff --git a/metering/handlers/freecall_handler.py b/metering/handlers/freecall_handler.py index f6b9e2f..95641a5 100644 --- a/metering/handlers/freecall_handler.py +++ b/metering/handlers/freecall_handler.py @@ -1,47 +1,41 @@ -import json -import logging - from constants import StatusCode, StatusMessage, HEADER_POST_RESPONSE from services import UsageService -from logger import setup_logger -from utils import validate_request, make_response +from logger import get_logger +from utils import validate_request, generate_lambda_response usage_service = UsageService() -setup_logger() -logger = logging.getLogger(__name__) +logger = get_logger(__name__) required_keys = ['organization_id', 'service_id'] def main(event, context): - if validate_request(required_keys, event['queryStringParameters']): - try: + logger.info("Free call request received") + try: + if validate_request(required_keys, event['queryStringParameters']): org_id = event['queryStringParameters']['organization_id'] service_id = event['queryStringParameters']['service_id'] username = event['queryStringParameters']['username'] + + logger.info(f"Fetched values from request \n" + f"username: {username} \n" + f"org_id: {org_id} \n" + f"service_id: {service_id} \n") + free_call_details = usage_service.get_free_call_details( username, org_id, service_id) - return_value = make_response( - status_code=StatusCode.SUCCESS_GET_CODE, - header=HEADER_POST_RESPONSE, - body=json.dumps(free_call_details) - ) - - except Exception as e: - logger.error(e) - return_value = make_response( - status_code=StatusCode.SERVER_ERROR_CODE, - header=HEADER_POST_RESPONSE, - body=json.dumps({"error": StatusMessage.SERVER_ERROR_MSG}) - ) - - else: - logger.error(f"Request validation failed for {event['queryStringParameters']}") - return_value = make_response( - status_code=StatusCode.BAD_PARAMETERS_CODE, - header=HEADER_POST_RESPONSE, - body=json.dumps({"error": StatusMessage.BAD_PARAMETER}) - ) - - return return_value + response = free_call_details + status_code = StatusCode.SUCCESS_GET_CODE + else: + logger.error(f"Request validation failed") + logger.info(event) + response = StatusMessage.BAD_PARAMETER + status_code = StatusCode.BAD_PARAMETERS_CODE + except Exception as e: + logger.error("Failed to get free call details") + logger.info(event) + status_code = StatusCode.SERVER_ERROR_CODE, + response = StatusMessage.SERVER_ERROR_MSG + + return generate_lambda_response(status_code, response) diff --git a/metering/handlers/usage_handler.py b/metering/handlers/usage_handler.py index 2e1bc72..8110574 100644 --- a/metering/handlers/usage_handler.py +++ b/metering/handlers/usage_handler.py @@ -2,43 +2,37 @@ import logging from constants import StatusCode, StatusMessage -from logger import setup_logger +from logger import get_logger from services import UsageService -from utils import validate_request, make_response, usage_record_add_verify_fields +from utils import validate_request, usage_record_add_verify_fields, generate_lambda_response usage_service = UsageService() -setup_logger() -logger = logging.getLogger(__name__) +logger = get_logger(__name__) required_keys = ["organization_id", "service_id", "username", 'usage_value', 'usage_type', 'service_method', 'group_id', 'status', 'start_time', 'end_time'] def main(event, context): - request_dict = json.loads(event['body']) - + logger.info("Usage record request received") try: + request_dict = json.loads(event['body']) if validate_request(required_keys, request_dict): usage_detail_dict = usage_record_add_verify_fields(request_dict) - print(f"usage record after modification: {usage_detail_dict}") + logging.info(f"usage record after modification: {usage_detail_dict}") usage_service.save_usage_details(usage_detail_dict) - response = make_response( - StatusCode.SUCCESS_GET_CODE, - json.dumps({"status": StatusMessage.SUCCESS_POST_CODE}) - ) + response = "success" + status_code = StatusCode.SUCCESS_GET_CODE else: - logger.error(f'Request validation failed {request_dict}') - response = make_response( - StatusCode.BAD_PARAMETERS_CODE, - json.dumps({"status": StatusMessage.BAD_PARAMETER}) - ) + logger.error(f"Request validation failed") + logger.info(event) + response = StatusMessage.BAD_PARAMETER + status_code = StatusCode.BAD_PARAMETERS_CODE except Exception as e: - logger.error(e) - logger.error(f'failed for request {request_dict}') - response = make_response( - StatusCode.SERVER_ERROR_CODE, - json.dumps({"status": StatusMessage.SERVER_ERROR_MSG}) - ) + logger.error("Failed to get free call details") + logger.info(event) + status_code = StatusCode.SERVER_ERROR_CODE, + response = StatusMessage.SERVER_ERROR_MSG - return response + return generate_lambda_response(status_code, response) diff --git a/metering/logger.py b/metering/logger.py index c39d5cb..e443a57 100644 --- a/metering/logger.py +++ b/metering/logger.py @@ -1,12 +1,17 @@ -import logging.config +import logging +import sys +FORMATTER = logging.Formatter("%(asctime)s %(levelname)s %(name)s %(message)s") -def setup_logger(): - logging.basicConfig( - level=logging.INFO, format='%(asctime)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S') +def get_console_handler(): + console_handler = logging.StreamHandler(sys.stdout) + console_handler.setFormatter(FORMATTER) + return console_handler -if __name__ == '__main__': - setup_logger() - logger = logging.getLogger() - logger.info("hi, it is test") +def get_logger(logger_name): + logger = logging.getLogger(logger_name) + logger.setLevel(logging.INFO) + logger.addHandler(get_console_handler()) + logger.propagate = False + return logger diff --git a/metering/repository/base_repository.py b/metering/repository/base_repository.py index da4211e..be1b326 100644 --- a/metering/repository/base_repository.py +++ b/metering/repository/base_repository.py @@ -4,7 +4,7 @@ from sqlalchemy.orm import sessionmaker from settings import DB_URL -engine = create_engine(DB_URL, echo=True) +engine = create_engine(DB_URL, echo=False) Session = sessionmaker(bind=engine) default_session = Session() diff --git a/metering/serverless.yml b/metering/serverless.yml index 3c8e133..b417a3d 100644 --- a/metering/serverless.yml +++ b/metering/serverless.yml @@ -54,7 +54,7 @@ functions: type: request arn: ${file(./config.${self:provider.stage}.json):SIGN_AUTHORIZER} identitySource: ${file(./config.${self:provider.stage}.json):SIGN_HEADERS} - name: verify_authorizer + name: sign_authorizer # resultTtlInSeconds: 100 cors: origin: ${self:custom.origin.${self:provider.stage}} @@ -88,7 +88,7 @@ functions: type: request arn: ${file(./config.${self:provider.stage}.json):SIGN_AUTHORIZER} identitySource: ${file(./config.${self:provider.stage}.json):SIGN_HEADERS} - name: freecall_authorizer + name: sign_authorizer # resultTtlInSeconds: 100 cors: origin: ${self:custom.origin.${self:provider.stage}} @@ -121,7 +121,7 @@ functions: type: request arn: ${file(./config.${self:provider.stage}.json):SIGN_AUTHORIZER} identitySource: ${file(./config.${self:provider.stage}.json):SIGN_HEADERS} - name: usage_authorizer + name: sign_authorizer # resultTtlInSeconds: 100 cors: origin: ${self:custom.origin.${self:provider.stage}} diff --git a/metering/services.py b/metering/services.py index 186d9d3..d922e7f 100644 --- a/metering/services.py +++ b/metering/services.py @@ -1,13 +1,13 @@ import json -import logging import boto3 as boto3 +from logger import get_logger from settings import CONTRACT_API_ARN, CONTRACT_API_STAGE from storage import DatabaseStorage from utils import is_free_call -logger = logging.getLogger(__name__) +logger = get_logger(__name__) class UsageService(object): @@ -17,6 +17,7 @@ def get_free_call_details(self, username, org_id, service_id, group_id=None): total_calls, free_calls = self.storage_service.get_usage_details( username, org_id, service_id, group_id) + logger.info(f"Free calls allowed: {free_calls}, Total calls made: {total_calls}") if not free_calls: free_calls = 0 if not total_calls: @@ -31,10 +32,14 @@ def get_free_call_details(self, username, org_id, service_id, group_id=None): def save_usage_details(self, usage_details_dict): # nedd to introduce entities when we enhance feature to this service right now directly using dicts if not is_free_call(usage_details_dict): + logger.info("Received usage record request for paid call") channel_id = usage_details_dict['channel_id'] group_id = usage_details_dict['group_id'] + user_address = APIUtilityService().get_user_address(group_id, channel_id) usage_details_dict['user_address'] = user_address + logger.info(f"fetched user address from contract api: {user_address}") + self.storage_service.add_usage_data(usage_details_dict) return @@ -52,6 +57,7 @@ def get_user_address(self, group_id, channel_id): } try: + logger.info(f"Calling contract api for user_address") response = self.lambda_client.invoke( FunctionName=CONTRACT_API_ARN, Payload=json.dumps(lambda_payload) diff --git a/metering/storage.py b/metering/storage.py index dd51083..be53f3e 100644 --- a/metering/storage.py +++ b/metering/storage.py @@ -1,12 +1,13 @@ import logging +from logger import get_logger from models import UserOrgGroupModel, UsageModel from repository.org_service_config_repository import OrgServiceRepo from repository.usage_repository import UsageRepository from repository.user_org_group_repository import UserOrgGroupRepository from constants import PAYMENT_MODE_FREECALL_VALUE, PAYMENT_MODE_ESCROW_VALUE -logger = logging.getLogger(__name__) +logger = get_logger(__name__) class Storage(object): @@ -110,8 +111,14 @@ def add_usage_data(self, usage_details): def get_usage_details(self, user_name, org_id, service_id, group_id=None): optin_time = self.usage_repo.get_optin_time( user_name, org_id, service_id) + logger.info(f"opt in for the user{user_name}, \n" + f"org_id: {org_id}, service_id: {service_id} \n" + f"{optin_time.strftime('%Y-%m-%d %H:%M:%S')}") + free_calls = self.org_service_config_repo.get_service_config( org_id, service_id, optin_time) + total_calls = self.usage_repo.get_total_calls( user_name, org_id, service_id) + return total_calls, free_calls diff --git a/metering/utils.py b/metering/utils.py index 3f0ba4e..db867cd 100644 --- a/metering/utils.py +++ b/metering/utils.py @@ -1,28 +1,25 @@ +import json import logging from constants import PAYMENT_MODE_FREECALL_VALUE -def make_response(status_code, body, header=None): - return { - "statusCode": status_code, - "headers": header, - "body": body +def generate_lambda_response(status_code, message, headers=None): + response = { + 'statusCode': status_code, + 'body': json.dumps(message), + 'headers': { + 'Content-Type': 'application/json', + "X-Requested-With": '*', + "Access-Control-Allow-Headers": 'Access-Control-Allow-Origin, Content-Type,X-Amz-Date,Authorization,' + 'X-Api-Key,x-requested-with', + "Access-Control-Allow-Origin": '*', + "Access-Control-Allow-Methods": 'GET,OPTIONS,POST' + } } - - -def configure_log(logger): - logger.setLevel(logging.INFO) - - # create a file handler - handler = logging.StreamHandler() - handler.setLevel(logging.INFO) - # create a logging format - formatter = logging.Formatter( - '%(asctime)s - %(name)s - %(levelname)s - %(message)s') - handler.setFormatter(formatter) - # add the handlers to the logger - logger.addHandler(handler) + if headers is not None: + response["headers"].update(headers) + return response def validate_request(required_keys, request_body): From 45fd4b2578bbeea1631e87f56482e7e9d994d6c5 Mon Sep 17 00:00:00 2001 From: Pratik Prajapati Date: Thu, 3 Oct 2019 11:11:17 +0530 Subject: [PATCH 2/3] added coverall, after removing it because of issue in server of coverall --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 25ab3c7..a2fcfe6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -52,7 +52,7 @@ jobs: coverage report -m mkdir ../test-reports coverage html -d ../test-reports -# COVERALLS_REPO_TOKEN=${COVERALLS_REPO_TOKEN} coveralls + COVERALLS_REPO_TOKEN=${COVERALLS_REPO_TOKEN} coveralls - store_test_results: path: test-reports - store_artifacts: From 137242c91288a52ec87ede9e4fad097dc9702dbd Mon Sep 17 00:00:00 2001 From: Pratik Prajapati Date: Thu, 3 Oct 2019 11:15:42 +0530 Subject: [PATCH 3/3] change in serverless --- metering/serverless.yml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/metering/serverless.yml b/metering/serverless.yml index b417a3d..ed426be 100644 --- a/metering/serverless.yml +++ b/metering/serverless.yml @@ -54,7 +54,7 @@ functions: type: request arn: ${file(./config.${self:provider.stage}.json):SIGN_AUTHORIZER} identitySource: ${file(./config.${self:provider.stage}.json):SIGN_HEADERS} - name: sign_authorizer + name: verify_authorizer # resultTtlInSeconds: 100 cors: origin: ${self:custom.origin.${self:provider.stage}} @@ -77,9 +77,6 @@ functions: subnetIds: - ${file(./config.${self:provider.stage}.json):VPC1} - ${file(./config.${self:provider.stage}.json):VPC2} - # The following are a few example events you can configure - # NOTE: Please make sure to change your handler code to work with those events - # Check the event documentation for details events: - http: path: /usage/freecalls @@ -88,7 +85,7 @@ functions: type: request arn: ${file(./config.${self:provider.stage}.json):SIGN_AUTHORIZER} identitySource: ${file(./config.${self:provider.stage}.json):SIGN_HEADERS} - name: sign_authorizer + name: freecall_authorizer # resultTtlInSeconds: 100 cors: origin: ${self:custom.origin.${self:provider.stage}} @@ -110,9 +107,6 @@ functions: subnetIds: - ${file(./config.${self:provider.stage}.json):VPC1} - ${file(./config.${self:provider.stage}.json):VPC2} - # The following are a few example events you can configure - # NOTE: Please make sure to change your handler code to work with those events - # Check the event documentation for details events: - http: path: /usage @@ -121,7 +115,7 @@ functions: type: request arn: ${file(./config.${self:provider.stage}.json):SIGN_AUTHORIZER} identitySource: ${file(./config.${self:provider.stage}.json):SIGN_HEADERS} - name: sign_authorizer + name: usage_authorizer # resultTtlInSeconds: 100 cors: origin: ${self:custom.origin.${self:provider.stage}}