Skip to content

Commit 9e3ed87

Browse files
gerrod3gerrod3
committed
Update Azure and S3 storage settings for main
Set new storage settings in template_config.yml based on pulpcore requirement (>=3.86.0,<3.86.0). This addresses the recent change where plugin-template stopped automatically setting test scenario storage settings and now requires plugins to define them in their template_config.yml.
1 parent a0479bc commit 9e3ed87

File tree

3 files changed

+31
-129
lines changed

3 files changed

+31
-129
lines changed

‎.ci/ansible/settings.py.j2‎

Lines changed: 0 additions & 124 deletions
Original file line numberDiff line numberDiff line change
@@ -26,127 +26,3 @@ API_ROOT = {{ api_root | repr }}
2626
{% endfor %}
2727
{% endif %}
2828

29-
{# =======================================
30-
Macros for legacy and new storage settings
31-
========================================== -#}
32-
33-
{%- macro s3_settings(legacy) -%}
34-
{%- if legacy %}
35-
DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage"
36-
AWS_ACCESS_KEY_ID = "{{ minio_access_key }}"
37-
AWS_SECRET_ACCESS_KEY = "{{ minio_secret_key }}"
38-
AWS_S3_REGION_NAME = "eu-central-1"
39-
AWS_S3_ADDRESSING_STYLE = "path"
40-
AWS_S3_SIGNATURE_VERSION = "s3v4"
41-
AWS_STORAGE_BUCKET_NAME = "pulp3"
42-
AWS_S3_ENDPOINT_URL = "http://minio:9000"
43-
AWS_DEFAULT_ACL = "@none None"
44-
{%- else %}
45-
STORAGES = {
46-
"default": {
47-
"BACKEND": "storages.backends.s3boto3.S3Boto3Storage",
48-
"OPTIONS": {
49-
"access_key": "{{ minio_access_key }}",
50-
"secret_key": "{{ minio_secret_key }}",
51-
"region_name": "eu-central-1",
52-
"addressing_style": "path",
53-
"signature_version": "s3v4",
54-
"bucket_name": "pulp3",
55-
"endpoint_url": "http://minio:9000",
56-
"default_acl": "@none None",
57-
},
58-
},
59-
"staticfiles": {
60-
"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage",
61-
},
62-
}
63-
{%- endif %}
64-
{%- endmacro -%}
65-
66-
{%- macro azure_settings(legacy) -%}
67-
{%- if legacy %}
68-
DEFAULT_FILE_STORAGE = "storages.backends.azure_storage.AzureStorage"
69-
AZURE_ACCOUNT_KEY = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
70-
AZURE_ACCOUNT_NAME = "devstoreaccount1"
71-
AZURE_CONTAINER = "pulp-test"
72-
AZURE_LOCATION = "pulp3"
73-
AZURE_OVERWRITE_FILES = True
74-
AZURE_URL_EXPIRATION_SECS = 120
75-
AZURE_CONNECTION_STRING = 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;'
76-
{%- else %}
77-
STORAGES = {
78-
"default": {
79-
"BACKEND": "storages.backends.azure_storage.AzureStorage",
80-
"OPTIONS": {
81-
"account_key": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==",
82-
"account_name": "devstoreaccount1",
83-
"location": "pulp3",
84-
"azure_container": "pulp-test",
85-
"overwrite_files": True,
86-
"expiration_secs": 120,
87-
"connection_string": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;",
88-
},
89-
},
90-
"staticfiles": {
91-
"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage",
92-
},
93-
}
94-
{%- endif %}
95-
{%- endmacro -%}
96-
97-
{%- macro gcp_settings(legacy) -%}
98-
{%- if legacy %}
99-
DEFAULT_FILE_STORAGE = "storages.backends.gcloud.GoogleCloudStorage"
100-
GS_BUCKET_NAME = "gcppulp"
101-
GS_CUSTOM_ENDPOINT = "http://ci-gcp:4443"
102-
{%- else %}
103-
STORAGES = {
104-
"default": {
105-
"BACKEND": "storages.backends.gcloud.GoogleCloudStorage",
106-
"OPTIONS": {
107-
"bucket_name": "gcppulp",
108-
"custom_endpoint": "http://ci-gcp:4443",
109-
},
110-
},
111-
"staticfiles": {
112-
"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage",
113-
},
114-
}
115-
{%- endif %}
116-
{%- endmacro -%}
117-
118-
{#- ==========================================
119-
Render according to test_storages_compat_layer
120-
==============================================
121-
122-
Case 1) test_storages_compat_layer is unset
123-
- All storages render the legacy setting
124-
- Branches using pulpcore <3.70 must leave this key unset (use legacy)
125-
126-
Case 2) test_storages_compat_layer is True
127-
- To tests both work, only one setting uses the new storage setting
128-
- Branches using pulpcore >=3.70,<3.85 must set this key to True (test both)
129-
130-
Case 3) test_storages_compat_layer is False
131-
- All storages render the new setting
132-
- Branches using pulpcore >=3.85 must set this key to False (use new)
133-
-#}
134-
135-
{% if s3_test | default(false) or azure_test | default(false) or gcp_test | default(false)%}
136-
MEDIA_ROOT=""
137-
{% endif %}
138-
{%- if test_storages_compat_layer is not defined -%}
139-
{%- if s3_test | default(false) -%}{{ s3_settings(legacy=True) }}{%- endif -%}
140-
{%- if azure_test | default(false) -%}{{ azure_settings(legacy=True) }}{%- endif -%}
141-
{%- if gcp_test | default(false) -%}{{ gcp_settings(legacy=True) }}{%- endif -%}
142-
{%- else -%}
143-
{%- if test_storages_compat_layer is true -%}
144-
{%- if s3_test | default(false) -%}{{ s3_settings(legacy=False) }}{%- endif -%}
145-
{%- if azure_test | default(false) -%}{{ azure_settings(legacy=True) }}{%- endif -%}
146-
{%- if gcp_test | default(false) -%}{{ gcp_settings(legacy=True) }}{%- endif -%}
147-
{%- elif test_storages_compat_layer is false -%}
148-
{%- if s3_test | default(false) -%}{{ s3_settings(legacy=False) }}{%- endif -%}
149-
{%- if azure_test | default(false) -%}{{ azure_settings(legacy=False) }}{%- endif -%}
150-
{%- if gcp_test | default(false) -%}{{ gcp_settings(legacy=False) }}{%- endif -%}
151-
{%- endif -%}
152-
{%- endif -%}

‎.github/workflows/scripts/install.sh‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -108,9 +108,8 @@ if [ "$TEST" = "s3" ]; then
108108
sed -i -e '$a s3_test: true\
109109
minio_access_key: "'$MINIO_ACCESS_KEY'"\
110110
minio_secret_key: "'$MINIO_SECRET_KEY'"\
111-
pulp_scenario_settings: {"DISABLED_authentication_backends": "@merge django.contrib.auth.backends.RemoteUserBackend", "DISABLED_authentication_json_header": "HTTP_X_RH_IDENTITY", "DISABLED_authentication_json_header_jq_filter": ".identity.user.username", "DISABLED_authentication_json_header_openapi_security_scheme": {"description": "External OAuth integration", "flows": {"clientCredentials": {"scopes": {"api.console": "grant_access_to_pulp"}, "tokenUrl": "https://your-identity-provider/token/issuer"}}, "type": "oauth2"}, "DISABLED_rest_framework__default_authentication_classes": "@merge pulpcore.app.authentication.JSONHeaderRemoteAuthentication", "domain_enabled": true, "hide_guarded_distributions": true, "rest_framework__default_permission_classes": ["pulpcore.plugin.access_policy.AccessPolicyFromSettings"], "spectacular_settings__oas_version": "3.0.3"}\
111+
pulp_scenario_settings: {"DISABLED_authentication_backends": "@merge django.contrib.auth.backends.RemoteUserBackend", "DISABLED_authentication_json_header": "HTTP_X_RH_IDENTITY", "DISABLED_authentication_json_header_jq_filter": ".identity.user.username", "DISABLED_authentication_json_header_openapi_security_scheme": {"description": "External OAuth integration", "flows": {"clientCredentials": {"scopes": {"api.console": "grant_access_to_pulp"}, "tokenUrl": "https://your-identity-provider/token/issuer"}}, "type": "oauth2"}, "DISABLED_rest_framework__default_authentication_classes": "@merge pulpcore.app.authentication.JSONHeaderRemoteAuthentication", "MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.s3boto3.S3Boto3Storage", "OPTIONS": {"access_key": "AKIAIT2Z5TDYPX3ARJBA", "addressing_style": "path", "bucket_name": "pulp3", "default_acl": "@none", "endpoint_url": "http://minio:9000", "region_name": "eu-central-1", "secret_key": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "signature_version": "s3v4"}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "domain_enabled": true, "hide_guarded_distributions": true, "rest_framework__default_permission_classes": ["pulpcore.plugin.access_policy.AccessPolicyFromSettings"], "spectacular_settings__oas_version": "3.0.3"}\
112112
pulp_scenario_env: {}\
113-
test_storages_compat_layer: false\
114113
' vars/main.yaml
115114
export PULP_API_ROOT="/rerouted/djnd/"
116115
fi
@@ -123,9 +122,8 @@ if [ "$TEST" = "azure" ]; then
123122
- ./azurite:/etc/pulp\
124123
command: "azurite-blob --blobHost 0.0.0.0"' vars/main.yaml
125124
sed -i -e '$a azure_test: true\
126-
pulp_scenario_settings: {"api_root_rewrite_header": "X-API-Root", "content_origin": null, "domain_enabled": true, "rest_framework__default_authentication_classes": "@merge pulpcore.app.authentication.PulpRemoteUserAuthentication", "rest_framework__default_permission_classes": ["pulpcore.plugin.access_policy.DefaultAccessPolicy"], "task_diagnostics": ["memory"]}\
125+
pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.azure_storage.AzureStorage", "OPTIONS": {"account_key": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", "account_name": "devstoreaccount1", "azure_container": "pulp-test", "connection_string": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;", "expiration_secs": 120, "location": "pulp3", "overwrite_files": true}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "api_root_rewrite_header": "X-API-Root", "content_origin": null, "domain_enabled": true, "rest_framework__default_authentication_classes": "@merge pulpcore.app.authentication.PulpRemoteUserAuthentication", "rest_framework__default_permission_classes": ["pulpcore.plugin.access_policy.DefaultAccessPolicy"], "task_diagnostics": ["memory"]}\
127126
pulp_scenario_env: {}\
128-
test_storages_compat_layer: false\
129127
' vars/main.yaml
130128
fi
131129

‎template_config.yml‎

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,20 @@ pulp_settings:
6363
tmpfile_protection_time: 10
6464
upload_protection_time: 10
6565
pulp_settings_azure:
66+
MEDIA_ROOT: ''
67+
STORAGES:
68+
default:
69+
BACKEND: storages.backends.azure_storage.AzureStorage
70+
OPTIONS:
71+
account_key: Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
72+
account_name: devstoreaccount1
73+
azure_container: pulp-test
74+
connection_string: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;
75+
expiration_secs: 120
76+
location: pulp3
77+
overwrite_files: true
78+
staticfiles:
79+
BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage
6680
api_root_rewrite_header: X-API-Root
6781
content_origin: null
6882
domain_enabled: true
@@ -85,6 +99,21 @@ pulp_settings_s3:
8599
tokenUrl: https://your-identity-provider/token/issuer
86100
type: oauth2
87101
DISABLED_rest_framework__default_authentication_classes: '@merge pulpcore.app.authentication.JSONHeaderRemoteAuthentication'
102+
MEDIA_ROOT: ''
103+
STORAGES:
104+
default:
105+
BACKEND: storages.backends.s3boto3.S3Boto3Storage
106+
OPTIONS:
107+
access_key: AKIAIT2Z5TDYPX3ARJBA
108+
addressing_style: path
109+
bucket_name: pulp3
110+
default_acl: '@none'
111+
endpoint_url: http://minio:9000
112+
region_name: eu-central-1
113+
secret_key: fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS
114+
signature_version: s3v4
115+
staticfiles:
116+
BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage
88117
domain_enabled: true
89118
hide_guarded_distributions: true
90119
rest_framework__default_permission_classes:
@@ -112,6 +141,5 @@ test_lowerbounds: true
112141
test_performance: false
113142
test_reroute: true
114143
test_s3: true
115-
test_storages_compat_layer: false
116144
use_issue_template: true
117145

0 commit comments

Comments
 (0)