From 4db98e7d612afde4b96e482ca26f381ccaa50d2a Mon Sep 17 00:00:00 2001 From: Deepak Devadathan Date: Fri, 3 Feb 2023 17:37:21 +1100 Subject: [PATCH 1/6] updated jets3t properties for spark provisioning Signed-off-by: Deepak Devadathan --- ansible/roles/analytics-spark-provision/templates/jets3t.j2 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ansible/roles/analytics-spark-provision/templates/jets3t.j2 b/ansible/roles/analytics-spark-provision/templates/jets3t.j2 index 4de8480bc2..829b6a82f1 100644 --- a/ansible/roles/analytics-spark-provision/templates/jets3t.j2 +++ b/ansible/roles/analytics-spark-provision/templates/jets3t.j2 @@ -1,5 +1,9 @@ storage-service.request-signature-version={{ jets3t_s3_request_signature_version }} -s3service.s3-endpoint={% if jets3t_s3_endpoint_host %}{{ jets3t_s3_endpoint_host }}{% else %}s3-ap-south-1.amazonaws.com{% endif %} +{% if jets3t_s3_endpoint_host %} +s3service.s3-endpoint={{ jets3t_s3_endpoint_host }} +{% else %} +s3service.s3-endpoint=s3-ap-south-1.amazonaws.com +{% endif %} s3service.disable-dns-buckets={{ jets3t_s3_disable_dns_buckets }} s3service.https-only={{ jets3t_s3_https_only }} {% if jets3t_s3_default_bucket_location %} From ea625349fae9c627ab782dcd77d41fb03637828b Mon Sep 17 00:00:00 2001 From: Deepak Devadathan Date: Fri, 3 Feb 2023 17:42:59 +1100 Subject: [PATCH 2/6] disabled update_cache Signed-off-by: Deepak Devadathan --- ansible/roles/analytics-bootstrap-always/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/analytics-bootstrap-always/tasks/main.yml b/ansible/roles/analytics-bootstrap-always/tasks/main.yml index 065a3b3afb..216bd61750 100644 --- a/ansible/roles/analytics-bootstrap-always/tasks/main.yml +++ b/ansible/roles/analytics-bootstrap-always/tasks/main.yml @@ -23,7 +23,7 @@ apt: name: "{{item}}" state: latest - update_cache: true + update_cache: false with_items: - rsync - zip From 9d2e83b32c192427f1bf5e8eeb76bc892feaad07 Mon Sep 17 00:00:00 2001 From: Deepak Devadathan Date: Fri, 3 Feb 2023 17:53:16 +1100 Subject: [PATCH 3/6] disabled update cache Signed-off-by: Deepak Devadathan --- ansible/roles/analytics-bootstrap-spark/tasks/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/roles/analytics-bootstrap-spark/tasks/main.yml b/ansible/roles/analytics-bootstrap-spark/tasks/main.yml index 663f76d68a..38531209b8 100644 --- a/ansible/roles/analytics-bootstrap-spark/tasks/main.yml +++ b/ansible/roles/analytics-bootstrap-spark/tasks/main.yml @@ -64,7 +64,7 @@ - name: Install required python packages become: yes - action: apt pkg={{ item }} state=present update_cache=yes + action: apt pkg={{ item }} state=present update_cache=no with_items: - libffi-dev - libssl-dev @@ -74,7 +74,7 @@ - name: Install libraries for spark bootstrap become: yes - action: apt pkg={{ item }} state=present update_cache=yes + action: apt pkg={{ item }} state=present update_cache=no with_items: - build-essential - git From decd6b4b3e2f13fffec970723b0da4f94ed88651 Mon Sep 17 00:00:00 2001 From: Deepak Devadathan Date: Fri, 3 Feb 2023 18:02:53 +1100 Subject: [PATCH 4/6] enabled cache Signed-off-by: Deepak Devadathan --- ansible/roles/analytics-bootstrap-spark/tasks/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/roles/analytics-bootstrap-spark/tasks/main.yml b/ansible/roles/analytics-bootstrap-spark/tasks/main.yml index 38531209b8..663f76d68a 100644 --- a/ansible/roles/analytics-bootstrap-spark/tasks/main.yml +++ b/ansible/roles/analytics-bootstrap-spark/tasks/main.yml @@ -64,7 +64,7 @@ - name: Install required python packages become: yes - action: apt pkg={{ item }} state=present update_cache=no + action: apt pkg={{ item }} state=present update_cache=yes with_items: - libffi-dev - libssl-dev @@ -74,7 +74,7 @@ - name: Install libraries for spark bootstrap become: yes - action: apt pkg={{ item }} state=present update_cache=no + action: apt pkg={{ item }} state=present update_cache=yes with_items: - build-essential - git From 50ef5993ca4a09825426b5527fbb730a23169f41 Mon Sep 17 00:00:00 2001 From: Deepak Devadathan Date: Fri, 3 Feb 2023 18:06:57 +1100 Subject: [PATCH 5/6] enabled cache Signed-off-by: Deepak Devadathan --- ansible/roles/analytics-bootstrap-always/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/analytics-bootstrap-always/tasks/main.yml b/ansible/roles/analytics-bootstrap-always/tasks/main.yml index 216bd61750..065a3b3afb 100644 --- a/ansible/roles/analytics-bootstrap-always/tasks/main.yml +++ b/ansible/roles/analytics-bootstrap-always/tasks/main.yml @@ -23,7 +23,7 @@ apt: name: "{{item}}" state: latest - update_cache: false + update_cache: true with_items: - rsync - zip From 4e613be6845b3d874c63e210fc33530410934214 Mon Sep 17 00:00:00 2001 From: Deepak Devadathan Date: Mon, 20 Feb 2023 10:15:23 +1100 Subject: [PATCH 6/6] added instance principal env to analytics user Added instance principal env variable remove configuration of oci cli as its will be using instance principal Signed-off-by: Deepak Devadathan --- .../analytics-bootstrap-spark/tasks/main.yml | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/ansible/roles/analytics-bootstrap-spark/tasks/main.yml b/ansible/roles/analytics-bootstrap-spark/tasks/main.yml index 663f76d68a..117ebe1e1e 100644 --- a/ansible/roles/analytics-bootstrap-spark/tasks/main.yml +++ b/ansible/roles/analytics-bootstrap-spark/tasks/main.yml @@ -10,6 +10,7 @@ with_items: - {var: 'azure_storage_key', value: '{{ sunbird_private_storage_account_name }}'} - {var: 'azure_storage_secret', value: '{{ sunbird_private_storage_account_key }}'} + - {var: 'OCI_CLI_AUTH', value: 'instance_principal'} - name: Adding PATH for oci cli Vars to bashrc file of spark. @@ -29,17 +30,17 @@ state: directory when: cloud_service_provider == "oci" -- name: Create OCI cli config location - become: yes - become_user: "{{ analytics_user }}" - template: src=oci-key.j2 dest={{ analytics_user_home }}/.oci/oci-key.pem mode=600 owner={{ analytics_user }} group={{ analytics_group }} - when: cloud_service_provider == "oci" +# - name: Create OCI cli config location +# become: yes +# become_user: "{{ analytics_user }}" +# template: src=oci-key.j2 dest={{ analytics_user_home }}/.oci/oci-key.pem mode=600 owner={{ analytics_user }} group={{ analytics_group }} +# when: cloud_service_provider == "oci" -- name: Create OCI cli config file - become: yes - become_user: "{{ analytics_user }}" - template: src=oci-cli-config.j2 dest={{ analytics_user_home }}/.oci/config mode=600 owner={{ analytics_user }} group={{ analytics_group }} - when: cloud_service_provider == "oci" +# - name: Create OCI cli config file +# become: yes +# become_user: "{{ analytics_user }}" +# template: src=oci-cli-config.j2 dest={{ analytics_user_home }}/.oci/config mode=600 owner={{ analytics_user }} group={{ analytics_group }} +# when: cloud_service_provider == "oci" - name: Adding ENV Vars to spark servers environment. become: yes