Merge branch 'master' of https://github.com/CodeForPhilly/paws-data-pipeline

Temp fix of phone number exception issue

Author: sposerina

src/server/Dockerfile

@@ -41,4 +41,7 @@ RUN set FLASK_APP=server/app.py
 RUN export FLASK_APP
 
 # This abomination ensures that the PG server has finished its restart cycle
-CMD echo "SLEEPING 10"; sleep 10; echo "WAKING"; alembic upgrade head ; python -m flask run --host=0.0.0.0
+CMD echo "SLEEPING 10"; sleep 10; echo "WAKING"; alembic upgrade head ; python -m flask run --host=0.0.0.0 --no-reload
+
+# --no-reload prevents Flask restart, which usually happens in middle of create_base_users()
+#TODO: SECURITY - ensure we are not running in debug mode in production

src/server/api/jwt_ops.py

@@ -6,7 +6,8 @@
     create_access_token,
     get_jwt_identity,
     verify_jwt_in_request,
-    get_jwt_claims,
+    get_jwt
+   
 )
 
 from app import app, jwt
@@ -16,28 +17,20 @@ def admin_required(fn):
     @wraps(fn)
     def wrapper(*args, **kwargs):
         verify_jwt_in_request()
-        claims = get_jwt_claims()
+        claims = get_jwt()
         if claims["role"] != "admin":  # TODO could be multiple
             return jsonify(msg="Admins only!"), 403
         else:
             return fn(*args, **kwargs)
 
     return wrapper
 
-
-@jwt.user_claims_loader
-def add_claims_to_access_token(accesslevel):
-    """ Adds role k/v to token """
-    return {"role": accesslevel}
-
-
 def create_token(username, accesslevel):
-    """ Create a JWT *access* token for the specified user and role. 
-        Role is magically added by the user_claims_loader decorator 
+    """ Create a JWT *access* token for the specified user ('sub:') and role ('role:'). 
     """
-    # Identity can be any data that is json serializable
-    new_token = create_access_token(identity=username)
-    # add_claims_to_access_token(accesslevel)
+    # Identity can be any data that is json serializable, we just use username
+    addl_claims = {'role': accesslevel}
+    new_token = create_access_token(identity=username, additional_claims=addl_claims)
     return jsonify(access_token=new_token)
 
 

src/server/api/user_api.py

@@ -67,50 +67,30 @@ def user_test_fail():
     return jsonify("Here's your failure"), 401
 
 
-@user_api.route("/api/user/login", methods=["POST"])
-def user_login():
-    """ Validate user in db, return JWT if legit and active.
-        Expects non-json form data
-    """
-
-    with engine.connect() as connection:
-
-        pwhash = None
-        s = text(
-            """select password, pdp_user_roles.role, active 
-                from pdp_users 
-                left join pdp_user_roles on pdp_users.role = pdp_user_roles._id 
-                where username=:u """
-        )
-        s = s.bindparams(u=request.form["username"])
-        result = connection.execute(s)
-
-        if result.rowcount:  # Did we get a match on username?
-            pwhash, role, is_active = result.fetchone()
-        else:
-            log_user_action(request.form["username"], "Failure", "Invalid username")
-            return jsonify("Bad credentials"), 401
-
-        if is_active.lower() == "y" and check_password(request.form["password"], pwhash):
-            # Yes, user is active and password matches
-            token = jwt_ops.create_token(request.form["username"], role)
-            log_user_action(request.form["username"], "Success", "Logged in")
-            return token
-
-        else:
-            log_user_action(request.form["username"], "Failure", "Bad password or inactive")
-            return jsonify("Bad credentials"), 401
-
 
 @user_api.route("/api/user/login_json", methods=["POST"])
 def user_login_json():
     """ Validate user in db, return JWT if legit and active.
         Expects json-encoded form data
     """
 
-    post_dict = json.loads(request.data)
-    username = post_dict["username"]
-    presentedpw = post_dict["password"]
+    def dummy_check():
+        """Perform a fake password hash check to take as much time as a real one."""
+        pw_bytes = bytes('password', "utf8")
+        check_password('password', pw_bytes)
+
+    try:
+        post_dict = json.loads(request.data)
+        username = post_dict["username"]
+        presentedpw = post_dict["password"]
+    except:
+        dummy_check()    # Take the same time as with well-formed requests 
+        return jsonify("Bad credentials"), 401
+
+    if not (isinstance(username, str) and isinstance(presentedpw, str) ):
+        dummy_check()  # Take the same time as with well-formed requests 
+        return jsonify("Bad credentials"), 401   # Don't give us ints, arrays, etc.
+
 
     with engine.connect() as connection:
 
@@ -128,6 +108,7 @@ def user_login_json():
             pwhash, role, is_active = result.fetchone()
         else:
             log_user_action(username, "Failure", "Invalid username")
+            dummy_check()
             return jsonify("Bad credentials"), 401
 
         if is_active.lower() == "y" and check_password(presentedpw, pwhash):
@@ -138,22 +119,23 @@ def user_login_json():
 
         else:
             log_user_action(username, "Failure", "Bad password or inactive")
+            # No dummy_check needed as we ran a real one to get here
             return jsonify("Bad credentials"), 401
 
 
 ###    Unexpired JWT required               ############################
 
 
 @user_api.route("/api/user/test_auth", methods=["GET"])
-@jwt_ops.jwt_required
+@jwt_ops.jwt_required()
 def user_test_auth():
     """ Liveness test, requires JWT """
     return jsonify(("OK from User Test - Auth  @" + str(datetime.now())))
 
 
 # Logout is not strictly needed; client can just delete JWT, but good for logging
 @user_api.route("/api/user/logout", methods=["POST"])
-@jwt_ops.jwt_required
+@jwt_ops.jwt_required()
 def user_logout():
     username = request.form["username"]  # TODO: Should be JSON all throughout
     # Log the request

src/server/app.py

@@ -4,16 +4,17 @@
 
 from flask_jwt_extended import JWTManager
 
+from secrets import JWT_SECRET, APP_SECRET_KEY
 
 app = Flask(__name__)
 
-app.config["JWT_SECRET_KEY"] = "super-secret"  # TODO: SECURITY  Change this!
+app.config["JWT_SECRET_KEY"] = JWT_SECRET  
 app.config["JWT_ACCESS_TOKEN_EXPIRES"] = 300  # Seconds for timeout. 60 for testing.
 jwt = JWTManager(app)
 
 
 # def create_app():
-app.secret_key = "1u9L#*&I3Ntc"  # TODO: SECURITY  Change this!
+app.secret_key = APP_SECRET_KEY  
 app.config["MAX_CONTENT_LENGTH"] = 500 * 1024 * 1024  # 500 Megs
 app.config["SEND_FILE_MAX_AGE_DEFAULT"] = 0
 from api.admin_api import admin_api

src/server/datasource_manager.py

@@ -71,16 +71,20 @@ def __clean_csv_headers(header):
 }
 
 
-def volgistics_address(index, street):
+def volgistics_address(street, index):
     result = ""
 
-    for item in street:
-        if isinstance(item, str):
-            if " " in item:
-                result = item.split()[index]
+    if isinstance(street, str):
+        if " " in street:
+            if index == 1:
+                result = " ".join(street.split()[1:])
+            else:
+                result = street.split()[index]
+
 
     return result
 
+
 def normalize_phone_number(number):
     if str(number) == 'nan':
         return ""
@@ -90,13 +94,14 @@ def normalize_phone_number(number):
         return ""
     return phonenumbers.format_number(parsed_number, phonenumbers.PhoneNumberFormat.NATIONAL)
 
+
 SOURCE_NORMALIZATION_MAPPING = {
     "salesforcecontacts": {
         "source_id": "contact_id",
         "first_name": "first_name",
         "last_name": "last_name",
         "email": "email",
-        "mobile": lambda df: df["mobile"].combine_first(df["phone"]).apply(normalize_phone_number),
+        "mobile": lambda df: df["mobile"].combine_first(df["phone"]),
         "street_and_number": "mailing_street",
         "apartment": "mailing_street",
         "city": "mailing_city",
@@ -118,7 +123,7 @@ def normalize_phone_number(number):
         "first_name": "firstname",
         "last_name": "lastname",
         "email": "email",
-        "mobile": lambda df: df["phone"].apply(normalize_phone_number),
+        "mobile": lambda df: df["phone"],
         "street_and_number": "street",
         "apartment": "apartment",
         "city": "city",
@@ -133,9 +138,9 @@ def normalize_phone_number(number):
         "first_name": "first_name",
         "last_name": "last_name",
         "email": "email",
-        "mobile": lambda df: df["cell"].combine_first(df["home"]).apply(normalize_phone_number),
-        "street_and_number": lambda df: volgistics_address(1, df["street_1"]),
-        "apartment": lambda df: volgistics_address(0, df["street_1"]),
+        "mobile": lambda df: df["cell"].combine_first(df["home"]),
+        "street_and_number": lambda df: df["street_1"].apply(volgistics_address, index=1),
+        "apartment": lambda df: df["street_1"].apply(volgistics_address, index=0),
         "city": "city",
         "state": "state",
         "zip": "zip",

src/server/pipeline/normalize_matching_data.py

@@ -10,7 +10,7 @@ xlrd==1.2.0  # currently used for xlsx, but we should consider adjusting code to
 openpyxl
 requests
 pytest
-flask-jwt-extended
+flask-jwt-extended==4.0.2
 alembic
 flask-cors
-phonenumbers
+phonenumbers