Add support for EMR 6.1 and PrestoSQL plugin

Author: Varun Rao

README.md

@@ -22,7 +22,7 @@ The repo contains code tied to [AWS Big Data Blog](https://aws.amazon.com/blogs/
 | Module|  Tag | Cloudformation stack | Apache Ranger Version | EMR Version | Supported Plugins|
 | -------| --- | --- | --- | --- |-------------------------------------------------------- |
 | V1 | [1.0](https://github.com/aws-samples/aws-emr-apache-ranger/tree/v1.0) | [![Foo](images/launch_stack.png)](https://console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks/new?stackName=EMRSecurityWithRangerBlogV1&templateURL=https://s3.amazonaws.com/aws-bigdata-blog/artifacts/aws-blog-emr-ranger/1.0/cloudformation/nestedstack.template) | Apache Ranger 1.0, 2.1 | emr-5.28.1, emr-5.29.0, emr-5.30.1| Hive 2.x, Hadoop 2.x, PrestoDB 0.227/0.232 (Presto plugin needs Ranger 2.0) | 
-| V1 | (work in progress) | [![Foo](images/launch_stack.png)](https://console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks/new?stackName=EMRSecurityWithRangerBlogV1&templateURL=https://s3.amazonaws.com/aws-bigdata-blog/artifacts/aws-blog-emr-ranger/1.1/cloudformation/nestedstack.template) | Apache Ranger 2.2 | emr-6.1.0 | Hive 3.x, Hadoop 3.x, PrestoSQL 338 OR PrestoDB 0.232 | 
+| V1 | [1.1](https://github.com/aws-samples/aws-emr-apache-ranger/tree/v1.1) | [![Foo](images/launch_stack.png)](https://console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks/new?stackName=EMRSecurityWithRangerBlogV1&templateURL=https://s3.amazonaws.com/aws-bigdata-blog/artifacts/aws-blog-emr-ranger/1.1/cloudformation/nestedstack.template) | Apache Ranger 2.2 | emr-5.29.0, emr-5.30.1, emr-6.1.0 | Hive 3.x, Hadoop 3.x, PrestoSQL 338 OR PrestoDB 0.232 | 
 
 > WARNING: The current V1 setup does not enable strong cluster level Auth (Kerberos) for EMR. Only LDAP enabled Hue UI. V2 will support Kerberos - refer to the [roadmap](https://github.com/aws-samples/aws-emr-apache-ranger/projects/1) for details.
 ### PrestoSQL Ranger plugin (EMR 6.1 & Ranger 2.2)

aws_emr_blog_v1/cloudformation/emr-template.template

@@ -59,7 +59,7 @@ Parameters:
       - emr-5.28.0
       - emr-5.28.1
       - emr-5.29.0
-      - emr-5.30.0
+      - emr-5.30.1
       - emr-5.31.0
       - emr-5.32.0
       - emr-6.1.0
@@ -146,6 +146,7 @@ Resources:
             Path: !Join ['', ['s3://', !Ref s3artifactsRepo, '/', !Ref s3artifactsRepoVersion, '/scripts/download-scripts.sh']]
             Args:
               - !Join ['', ['s3://', !Ref s3artifactsRepo]]
+              - !Ref 's3artifactsRepoVersion'
       Configurations:
         - Classification: hue-ini
           Configurations:

aws_emr_blog_v1/cloudformation/nestedstack.template

@@ -87,22 +87,13 @@ Parameters:
     Type: String
     Default: '2.0'
   emrReleaseLabel:
-    Default: emr-5.29.0
+    Default: emr-6.1.0
     AllowedValues:
-      - emr-5.0.0
-      - emr-5.4.0
-      - emr-5.16.0
-      - emr-5.17.0
-      - emr-5.26.0
-      - emr-5.27.0
-      - emr-5.28.0
-      - emr-5.28.1
       - emr-5.29.0
-      - emr-5.30.0
+      - emr-5.30.1
       - emr-5.31.0
-      - emr-5.32.0
       - emr-6.1.0
-    Description: Release label for the EMR cluster
+    Description: Release label for the EMR clusterPrestoEngine
     Type: String
   PrestoEngine:
     Description: Presto Engine. PrestoSQL is only available with EMR 6.x
@@ -114,7 +105,7 @@ Parameters:
     Description: S3 location of the repo.
     Type: String
   s3artifactsRepoVersion:
-    Default: 1.0
+    Default: 1.1
     Description: Project version
     Type: String
     AllowedValues:

aws_emr_blog_v1/cloudformation/ranger-server.template

@@ -124,7 +124,7 @@ Parameters:
       - emr-5.28.0
       - emr-5.28.1
       - emr-5.29.0
-      - emr-5.30.0
+      - emr-5.30.1
       - emr-5.31.0
       - emr-5.32.0
       - emr-6.1.0
@@ -412,7 +412,7 @@ Resources:
             '
             - 'aws s3 cp '
             - !Join ['', ['s3://', !Ref s3artifactsRepo, '/', !Ref s3artifactsRepoVersion]]
-            - '/scripts/install-ranger-admin-server.sh .
+            - '/scripts/install-ranger-admin-server.sh . --region us-east-1
 
             '
             - 'yum update aws-cfn-bootstrap

aws_emr_blog_v1/inputdata/ranger-presto-policy-analyst1.json

@@ -56,10 +56,6 @@
      "type": "alter",
      "isAllowed": true
     },
-    {
-     "type": "admin",
-     "isAllowed": true
-    },
     {
      "type": "all",
      "isAllowed": true
@@ -101,4 +97,4 @@
  ],
  "zoneName":"",
  "isDenyAllElse":false
-}
+}

aws_emr_blog_v1/inputdata/ranger-presto-policy-analyst2.json

@@ -56,10 +56,6 @@
      "type": "alter",
      "isAllowed": true
     },
-    {
-     "type": "admin",
-     "isAllowed": true
-    },
     {
      "type": "all",
      "isAllowed": true
@@ -101,4 +97,4 @@
  ],
  "zoneName":"",
  "isDenyAllElse":false
-}
+}

aws_emr_blog_v1/inputdata/ranger-presto-policy-general.json

@@ -50,10 +50,6 @@
      "type":"alter",
      "isAllowed":true
     },
-    {
-     "type":"admin",
-     "isAllowed":true
-    },
     {
      "type":"all",
      "isAllowed":true
@@ -103,4 +99,4 @@
  ],
  "zoneName":"",
  "isDenyAllElse":false
-}
+}

aws_emr_blog_v1/inputdata/ranger-presto-policy-information-schema.json

@@ -64,10 +64,6 @@
      "type":"alter",
      "isAllowed":true
     },
-    {
-     "type":"admin",
-     "isAllowed":true
-    },
     {
      "type":"all",
      "isAllowed":true
@@ -116,4 +112,4 @@
  ],
  "zoneName":"",
  "isDenyAllElse":false
-}
+}

aws_emr_blog_v1/scripts/download-scripts.sh

@@ -2,9 +2,10 @@
 set -euo pipefail
 set -x
 scripts_repo_path=$1
+project_version=${2-'1.0'}
 mkdir -p /tmp/aws-blog-emr-ranger/scripts/emr-steps
 cd /tmp/aws-blog-emr-ranger/scripts/emr-steps
 #sudo yum -y install svn
 #svn export $git_repo_path aws-blog-emr-ranger
-aws s3 sync $scripts_repo_path/scripts/emr-steps . --region us-east-1
-chmod -R 777 /tmp/aws-blog-emr-ranger
+aws s3 sync $scripts_repo_path/$project_version/scripts/emr-steps . --region us-east-1
+chmod -R 777 /tmp/aws-blog-emr-ranger

aws_emr_blog_v1/scripts/install-ranger-admin-server.sh

@@ -38,9 +38,9 @@ mysql_jar=mysql-connector-java-5.1.39.jar
 # Setup
 yum install -y openldap openldap-clients openldap-servers
 # Setup LDAP users
-aws s3 cp $s3path/$project_version/inputdata/load-users-new.ldf .
-aws s3 cp $s3path/$project_version/inputdata/modify-users-new.ldf .
-aws s3 cp $s3path/$project_version/scripts/create-users-using-ldap.sh .
+aws s3 cp $s3path/$project_version/inputdata/load-users-new.ldf . --region us-east-1
+aws s3 cp $s3path/$project_version/inputdata/modify-users-new.ldf . --region us-east-1
+aws s3 cp $s3path/$project_version/scripts/create-users-using-ldap.sh . --region us-east-1
 chmod +x create-users-using-ldap.sh
 ./create-users-using-ldap.sh $ldap_ip_address $ldap_admin_password $ldap_bind_password $ldap_default_user_password || true
 #Install mySQL
@@ -55,10 +55,10 @@ mysql -u root -prangeradmin -e "FLUSH PRIVILEGES;" || true
 rm -rf $installpath
 mkdir -p $installpath/hadoop
 cd $installpath
-aws s3 cp $ranger_s3path/$ranger_admin_server.tar.gz .
-aws s3 cp $ranger_s3path/$ranger_user_sync.tar.gz .
-aws s3 cp $mysql_jar_location .
-aws s3 cp $ranger_s3path/solr_for_audit_setup.tar.gz .
+aws s3 cp $ranger_s3path/$ranger_admin_server.tar.gz . --region us-east-1
+aws s3 cp $ranger_s3path/$ranger_user_sync.tar.gz . --region us-east-1
+aws s3 cp $mysql_jar_location . --region us-east-1
+aws s3 cp $ranger_s3path/solr_for_audit_setup.tar.gz . --region us-east-1
 #Update ranger admin install.properties
 tar -xvf $ranger_admin_server.tar.gz
 cd $ranger_admin_server