Merge pull request #37 from singnet/development

prashantramangupta · web-flow · commit 145e0596fea8 · 2019-08-28T14:07:41.000+05:30
metering changes to capture more metrics
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -0,0 +1,54 @@
+---
+version: 2
+jobs:
+  build:
+    docker:
+      - image: circleci/python:3.6.6-node
+      - image: circleci/mysql:8.0.16
+        environment:
+          MYSQL_ROOT_PASSWORD: rootpw
+          MYSQL_DATABASE: test_db
+          MYSQL_USER: user
+          MYSQL_PASSWORD: passw0rd
+    working_directory: ~/singnet
+    environment:
+      PYTHONPATH: ~/singnet/snet-platform-usage/metering
+    steps:
+      - checkout
+      - run:
+      # Our primary container isn't MYSQL so run a sleep command until it's ready.
+          name: Waiting for MySQL to be ready
+          command: |
+            for i in `seq 1 30`;
+            do
+              nc -z 127.0.0.1 3306 && echo Success && exit 0
+              echo -n .
+              sleep 1
+            done
+            echo Failed waiting for MySQL && exit 1
+      - run:
+          name: Initialise tables.
+          command: |
+            cd metering
+            cp tests/alembic.ini .
+            cp tests/test-settings.py settings.py
+            sudo pip install alembic
+            sudo pip install pymysql
+            sudo pip install cryptography
+            PYTHONPATH=$PWD
+            alembic upgrade head
+      - run:
+          name: Install dependencies & compile
+          command: |
+            cd metering
+            python -m compileall *.py
+            python -m compileall */*.py
+            sudo pip install -r requirements.txt
+      - run:
+          name: run tests for metering
+          command: |
+            cd metering
+            PYTHONPATH=$PWD python tests/test_total_calls.py
+      - store_artifacts:
+          path: test-reports
+          destination: test-reports
diff --git a/README.md b/README.md
@@ -1 +1,2 @@
-# snet-monitoring
+# Snet-Platform-Usage
+[![CircleCI](https://circleci.com/gh/singnet/snet-platform-usage.svg?style=svg)](https://circleci.com/gh/singnet/snet-platform-usage)
diff --git a/metering/alembic/versions/66d97ed3bdc5_usage_table_update.py b/metering/alembic/versions/66d97ed3bdc5_usage_table_update.py
@@ -0,0 +1,57 @@
+"""usage table update
+
+Revision ID: 66d97ed3bdc5
+Revises: 9ecbd29ee907
+Create Date: 2019-08-20 16:00:09.306326
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '66d97ed3bdc5'
+down_revision = '9ecbd29ee907'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('usage_table', sa.Column('channel_id', sa.VARCHAR(length=225), nullable=True))
+    op.add_column('usage_table', sa.Column('client_type', sa.VARCHAR(length=225), nullable=True))
+    op.add_column('usage_table', sa.Column('error_message', sa.VARCHAR(length=225), nullable=True))
+    op.add_column('usage_table', sa.Column('ethereum_json_rpc_endpoint', sa.VARCHAR(length=225), nullable=True))
+    op.add_column('usage_table', sa.Column('group_id', sa.VARCHAR(length=225), nullable=True))
+    op.add_column('usage_table', sa.Column('operation', sa.VARCHAR(length=225), nullable=True))
+    op.add_column('usage_table', sa.Column('org_id', sa.VARCHAR(length=225), nullable=True))
+    op.add_column('usage_table', sa.Column('registry_address_key', sa.VARCHAR(length=225), nullable=True))
+    op.add_column('usage_table', sa.Column('request_id', sa.VARCHAR(length=225), nullable=True))
+    op.add_column('usage_table', sa.Column('resource', sa.VARCHAR(length=225), nullable=True))
+    op.add_column('usage_table', sa.Column('response_code', sa.VARCHAR(length=225), nullable=True))
+    op.add_column('usage_table', sa.Column('response_time', sa.FLOAT(), nullable=True))
+    op.add_column('usage_table', sa.Column('user_address', sa.VARCHAR(length=225), nullable=True))
+    op.add_column('usage_table', sa.Column('user_details', sa.VARCHAR(length=225), nullable=True))
+    op.add_column('usage_table', sa.Column('username', sa.VARCHAR(length=225), nullable=True))
+    op.add_column('usage_table', sa.Column('version', sa.VARCHAR(length=225), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('usage_table', 'version')
+    op.drop_column('usage_table', 'username')
+    op.drop_column('usage_table', 'user_details')
+    op.drop_column('usage_table', 'user_address')
+    op.drop_column('usage_table', 'response_time')
+    op.drop_column('usage_table', 'response_code')
+    op.drop_column('usage_table', 'resource')
+    op.drop_column('usage_table', 'request_id')
+    op.drop_column('usage_table', 'registry_address_key')
+    op.drop_column('usage_table', 'org_id')
+    op.drop_column('usage_table', 'operation')
+    op.drop_column('usage_table', 'group_id')
+    op.drop_column('usage_table', 'ethereum_json_rpc_endpoint')
+    op.drop_column('usage_table', 'error_message')
+    op.drop_column('usage_table', 'client_type')
+    op.drop_column('usage_table', 'channel_id')
+    # ### end Alembic commands ###
diff --git a/metering/alembic/versions/b0f9443a30de_user_address_added.py b/metering/alembic/versions/b0f9443a30de_user_address_added.py
@@ -0,0 +1,38 @@
+"""user address added
+
+Revision ID: b0f9443a30de
+Revises: 66d97ed3bdc5
+Create Date: 2019-08-23 18:32:06.447515
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+
+# revision identifiers, used by Alembic.
+revision = 'b0f9443a30de'
+down_revision = '66d97ed3bdc5'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('usage_table', sa.Column('payment_mode', sa.VARCHAR(length=225), nullable=True))
+    op.add_column('usage_table', sa.Column('service_id', sa.VARCHAR(length=225), nullable=True))
+    op.add_column('user_org_group', sa.Column('user_address', sa.VARCHAR(length=225), nullable=True))
+    op.alter_column('user_org_group', 'user_name',
+                    existing_type=mysql.VARCHAR(length=225),
+                    nullable=True)
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('user_org_group', 'user_name',
+                    existing_type=mysql.VARCHAR(length=225),
+                    nullable=False)
+    op.drop_column('user_org_group', 'user_address')
+    op.drop_column('usage_table', 'service_id')
+    op.drop_column('usage_table', 'payment_mode')
+    # ### end Alembic commands ###
diff --git a/metering/alembic/versions/eeec29a1af7b_added_tables.py b/metering/alembic/versions/eeec29a1af7b_added_tables.py
@@ -27,8 +27,8 @@ def upgrade():
                     sa.Column('free_calls', sa.Integer(), nullable=False),
                     sa.Column('effective_start_date', sa.TIMESTAMP(
                         timezone=True), nullable=True),
-                    sa.Column('effective_end_date', sa.TIMESTAMP(timezone=True),
-                              server_default=sa.text('CURRENT_TIMESTAMP'), nullable=True),
+                    sa.Column('effective_end_date', sa.TIMESTAMP(
+                        timezone=True), nullable=True),
                     sa.Column('created_at', sa.TIMESTAMP(
                         timezone=True), nullable=True),
                     sa.PrimaryKeyConstraint('id')
diff --git a/metering/constants.py b/metering/constants.py
@@ -10,6 +10,7 @@ class StatusCode:
     "Access-Control-Allow-Origin": "*",
     "Access-Control-Allow-Methods": "OPTIONS,POST,GET"
 }
+PAYMENT_MODE_FREE_CALL = 'freecall'
 
 
 class StatusMessage:
diff --git a/metering/handlers/freecall_handler.py b/metering/handlers/freecall_handler.py
@@ -11,7 +11,7 @@
 setup_logger()
 logger = logging.getLogger(__name__)
 
-required_keys = ['username', 'organization_id', 'service_id']
+required_keys = ['organization_id', 'service_id']
 
 
 def main(event, context):
@@ -37,7 +37,7 @@ def main(event, context):
             )
 
     else:
-        logger.error('Request validation failed')
+        logger.error(f"Request validation failed for {event['queryStringParameters']}")
         return_value = make_response(
             status_code=StatusCode.BAD_PARAMETERS_CODE,
             header=HEADER_POST_RESPONSE,
diff --git a/metering/handlers/usage_handler.py b/metering/handlers/usage_handler.py
@@ -15,11 +15,31 @@
                  'service_method', 'group_id', 'status', 'start_time', 'end_time']
 
 
+def add_verify_fields(usage_detail_dict):
+    new_requred_keys = {
+        'usage_type', 'status', 'usage_value', 'start_time', 'end_time',
+        'created_at', 'payment_mode', 'group_id', 'registry_address_key',
+        'ethereum_json_rpc_endpoint', 'response_time', 'response_code', 'error_message',
+        'version', 'client_type', 'user_details', 'channel_id', 'operation', 'user_address',
+        'username', 'org_id', 'service_id', 'resource', 'request_id'
+    }
+    for key in new_requred_keys:
+        if key not in usage_detail_dict:
+            usage_detail_dict[key] = None
+
+    if usage_detail_dict['username'] is not None and usage_detail_dict['user_address'] is None:
+        usage_detail_dict['payment_mode'] = 'freecall'
+    else:
+        usage_detail_dict['payment_mode'] = 'paid'
+    return usage_detail_dict
+
+
 def main(event, context):
     usage_detail_dict = json.loads(event['body'])
 
     try:
         if validate_request(required_keys, usage_detail_dict):
+            usage_detail_dict = add_verify_fields(usage_detail_dict)
             usage_service.save_usage_details(usage_detail_dict)
             response = make_response(
                 StatusCode.SUCCESS_GET_CODE,
diff --git a/metering/models.py b/metering/models.py
@@ -1,4 +1,4 @@
-from sqlalchemy import Column, Integer, VARCHAR, ForeignKey, TIMESTAMP, func
+from sqlalchemy import Column, Integer, VARCHAR, ForeignKey, TIMESTAMP, func, FLOAT, String
 
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import relationship
@@ -24,7 +24,8 @@ class UserOrgGroupModel(Base):
     id = Column('id', Integer, primary_key=True)
     payment_group_id = Column('group', VARCHAR(225))
     org_id = Column('org_id', VARCHAR(225), nullable=False)
-    user_name = Column('user_name', VARCHAR(225), nullable=False)
+    user_name = Column('user_name', VARCHAR(225))
+    user_address = Column('user_address', VARCHAR(225))
     created_at = Column('created_at', TIMESTAMP(
         timezone=True), nullable=False, server_default=func.current_timestamp())
     service_id = Column('service_id', VARCHAR(225), nullable=False)
@@ -43,3 +44,21 @@ class UsageModel(Base):
     end_time = Column('end_time', TIMESTAMP(timezone=True))
     created_at = Column('created_at', TIMESTAMP(
         timezone=True), nullable=False, server_default=func.current_timestamp())
+    payment_mode = Column('payment_mode', VARCHAR(225))
+    group_id = Column('group_id', VARCHAR(225))
+    registry_address_key = Column('registry_address_key', VARCHAR(225))
+    ethereum_json_rpc_endpoint = Column('ethereum_json_rpc_endpoint', VARCHAR(225))
+    response_time = Column('response_time', FLOAT)
+    response_code = Column('response_code', VARCHAR(225))
+    error_message = Column('error_message', VARCHAR(225))
+    version = Column('version', VARCHAR(225))
+    client_type = Column('client_type', VARCHAR(225))
+    user_details = Column('user_details', VARCHAR(225))
+    channel_id = Column('channel_id', VARCHAR(225))
+    operation = Column('operation', VARCHAR(225))
+    user_address = Column('user_address', VARCHAR(225))
+    user_name = Column('username', VARCHAR(225))
+    org_id = Column('org_id', VARCHAR(225))
+    service_id = Column('service_id', VARCHAR(225))
+    resource = Column('resource', VARCHAR(225))
+    request_id = Column('request_id', VARCHAR(225))
diff --git a/metering/repository/user_org_group_repository.py b/metering/repository/user_org_group_repository.py
@@ -3,18 +3,26 @@
 
 
 class UserOrgGroupRepository(BaseRepository):
-    def get_user_org_group_data(self, payment_group_id, org_id, user_name, service_id, resource):
+
+    def get_user_org_group_id_by_username(self, username, org_id, service_id, resource):
         session = self.get_default_session()
-        user_org_query = session.query(UserOrgGroupModel) \
-            .filter(UserOrgGroupModel.resource == resource) \
+        user_org_query = session.query(UserOrgGroupModel).filter(UserOrgGroupModel.user_name == username) \
             .filter(UserOrgGroupModel.service_id == service_id) \
-            .filter(UserOrgGroupModel.user_name == user_name) \
+            .filter(UserOrgGroupModel.resource == resource) \
             .filter(UserOrgGroupModel.org_id == org_id)
 
-        if payment_group_id is not None:
-            user_org_query = user_org_query.filter(
-                UserOrgGroupModel.payment_group_id == payment_group_id)
+        user_org_group_data = user_org_query.first()
+        session.commit()
+        session.flush()
+        return user_org_group_data
 
+    def get_user_org_group_id_by_user_address(self, user_address, org_id, service_id, resource, payment_group_id):
+        session = self.get_default_session()
+        user_org_query = session.query(UserOrgGroupModel).filter(UserOrgGroupModel.user_address == user_address) \
+            .filter(UserOrgGroupModel.service_id == service_id) \
+            .filter(UserOrgGroupModel.resource == resource) \
+            .filter(UserOrgGroupModel.org_id == org_id) \
+            .filter(UserOrgGroupModel.payment_group_id == payment_group_id)
         user_org_group_data = user_org_query.first()
         session.commit()
         session.flush()
diff --git a/metering/requirements.txt b/metering/requirements.txt
@@ -1,8 +1,10 @@
 alembic==1.0.11
+boto3==1.9.216
 Mako==1.1.0
 MarkupSafe==1.1.1
 PyMySQL==0.9.3
 python-dateutil==2.8.0
 python-editor==1.0.4
 six==1.12.0
-SQLAlchemy==1.3.6
+SQLAlchemy==1.3.6
+requests==2.22.0
diff --git a/metering/services.py b/metering/services.py
@@ -1,4 +1,13 @@
+import json
+import logging
+
+import boto3 as boto3
+
+from settings import CONTRACT_API_ARN, CONTRACT_API_STAGE
 from storage import DatabaseStorage
+from utils import is_free_call
+
+logger = logging.getLogger(__name__)
 
 
 class UsageService(object):
@@ -13,10 +22,42 @@ def get_free_call_details(self, username, org_id, service_id, group_id=None):
         if not total_calls:
             total_calls = 0
 
-        return {"username": username, "org_id": org_id, "service_id": service_id, "total_calls_made": total_calls,
-                "free_calls_allowed": free_calls}
+        response = {"username": username, "org_id": org_id, "service_id": service_id, "total_calls_made": total_calls,
+                    "free_calls_allowed": free_calls}
+
+        logger.info(response)
+        return response
 
     def save_usage_details(self, usage_details_dict):
         # nedd to introduce entities when we enhance  feature to this service right now directly using dicts
+        if is_free_call(usage_details_dict):
+            channel_id = usage_details_dict['channel_id']
+            group_id = usage_details_dict['group_id']
+            user_address = APIUtilityService().get_user_address(group_id, channel_id)
+            usage_details_dict['user_address'] = user_address
         self.storage_service.add_usage_data(usage_details_dict)
         return
+
+
+class APIUtilityService:
+    lambda_client = boto3.client('lambda')
+
+    def get_user_address(self, group_id, channel_id):
+        lambda_payload = {
+            "httpMethod": "GET",
+            "requestContext": {"stage": CONTRACT_API_STAGE},
+            "path": f"/contract-api/group/{group_id}/channel/{channel_id}"
+        }
+        response = self.lambda_client.invoke(
+            FunctionName=CONTRACT_API_ARN,
+            Payload=json.dumps(lambda_payload)
+        )
+        response_body_raw = json.loads(
+            response.get('Payload').read())['body']
+        response_body = json.loads(response_body_raw)
+        try:
+            user_address = response_body['data'][0]['sender']
+        except Exception as e:
+            print(e)
+            raise Exception("Failed to get user address from marketplace")
+        return user_address
diff --git a/metering/storage.py b/metering/storage.py
diff --git a/metering/tests/alembic.ini b/metering/tests/alembic.ini
diff --git a/metering/tests/test-settings.py b/metering/tests/test-settings.py
diff --git a/metering/tests/test_contract_api.py b/metering/tests/test_contract_api.py
diff --git a/metering/tests/test_total_calls.py b/metering/tests/test_total_calls.py
diff --git a/metering/utils.py b/metering/utils.py

Original file line number	Diff line number	Diff line change
`@@ -1 +1,2 @@`
`1`		`-# snet-monitoring`
	`1`	`+# Snet-Platform-Usage`
	`2`	`+[![CircleCI](https://circleci.com/gh/singnet/snet-platform-usage.svg?style=svg)](https://circleci.com/gh/singnet/snet-platform-usage)`
Original file line number	Diff line number	Diff line change
`@@ -10,6 +10,7 @@ class StatusCode:`
`10`	`10`	`"Access-Control-Allow-Origin": "*",`
`11`	`11`	`"Access-Control-Allow-Methods": "OPTIONS,POST,GET"`
`12`	`12`	`}`
	`13`	`+PAYMENT_MODE_FREE_CALL = 'freecall'`
`13`	`14`
`14`	`15`
`15`	`16`	`class StatusMessage:`