Skip to content

Commit c52bf4b

Browse files
gerrod3gerrod3
committed
Update Azure and S3 storage settings for main
Set storage settings in template_config.yml based on pulpcore requirement (>=3.73.2,<3.100). Storage formats: Azure (new STORAGES), S3 (new STORAGES) This addresses the recent change where plugin-template stopped automatically setting test scenario storage settings and now requires plugins to define them in their template_config.yml.
1 parent 9963406 commit c52bf4b

File tree

4 files changed

+32
-130
lines changed

4 files changed

+32
-130
lines changed

‎.ci/ansible/settings.py.j2‎

Lines changed: 0 additions & 124 deletions
Original file line numberDiff line numberDiff line change
@@ -26,127 +26,3 @@ API_ROOT = {{ api_root | repr }}
2626
{% endfor %}
2727
{% endif %}
2828

29-
{# =======================================
30-
Macros for legacy and new storage settings
31-
========================================== -#}
32-
33-
{%- macro s3_settings(legacy) -%}
34-
{%- if legacy %}
35-
DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage"
36-
AWS_ACCESS_KEY_ID = "{{ minio_access_key }}"
37-
AWS_SECRET_ACCESS_KEY = "{{ minio_secret_key }}"
38-
AWS_S3_REGION_NAME = "eu-central-1"
39-
AWS_S3_ADDRESSING_STYLE = "path"
40-
AWS_S3_SIGNATURE_VERSION = "s3v4"
41-
AWS_STORAGE_BUCKET_NAME = "pulp3"
42-
AWS_S3_ENDPOINT_URL = "http://minio:9000"
43-
AWS_DEFAULT_ACL = "@none None"
44-
{%- else %}
45-
STORAGES = {
46-
"default": {
47-
"BACKEND": "storages.backends.s3boto3.S3Boto3Storage",
48-
"OPTIONS": {
49-
"access_key": "{{ minio_access_key }}",
50-
"secret_key": "{{ minio_secret_key }}",
51-
"region_name": "eu-central-1",
52-
"addressing_style": "path",
53-
"signature_version": "s3v4",
54-
"bucket_name": "pulp3",
55-
"endpoint_url": "http://minio:9000",
56-
"default_acl": "@none None",
57-
},
58-
},
59-
"staticfiles": {
60-
"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage",
61-
},
62-
}
63-
{%- endif %}
64-
{%- endmacro -%}
65-
66-
{%- macro azure_settings(legacy) -%}
67-
{%- if legacy %}
68-
DEFAULT_FILE_STORAGE = "storages.backends.azure_storage.AzureStorage"
69-
AZURE_ACCOUNT_KEY = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
70-
AZURE_ACCOUNT_NAME = "devstoreaccount1"
71-
AZURE_CONTAINER = "pulp-test"
72-
AZURE_LOCATION = "pulp3"
73-
AZURE_OVERWRITE_FILES = True
74-
AZURE_URL_EXPIRATION_SECS = 120
75-
AZURE_CONNECTION_STRING = 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;'
76-
{%- else %}
77-
STORAGES = {
78-
"default": {
79-
"BACKEND": "storages.backends.azure_storage.AzureStorage",
80-
"OPTIONS": {
81-
"account_key": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==",
82-
"account_name": "devstoreaccount1",
83-
"location": "pulp3",
84-
"azure_container": "pulp-test",
85-
"overwrite_files": True,
86-
"expiration_secs": 120,
87-
"connection_string": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;",
88-
},
89-
},
90-
"staticfiles": {
91-
"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage",
92-
},
93-
}
94-
{%- endif %}
95-
{%- endmacro -%}
96-
97-
{%- macro gcp_settings(legacy) -%}
98-
{%- if legacy %}
99-
DEFAULT_FILE_STORAGE = "storages.backends.gcloud.GoogleCloudStorage"
100-
GS_BUCKET_NAME = "gcppulp"
101-
GS_CUSTOM_ENDPOINT = "http://ci-gcp:4443"
102-
{%- else %}
103-
STORAGES = {
104-
"default": {
105-
"BACKEND": "storages.backends.gcloud.GoogleCloudStorage",
106-
"OPTIONS": {
107-
"bucket_name": "gcppulp",
108-
"custom_endpoint": "http://ci-gcp:4443",
109-
},
110-
},
111-
"staticfiles": {
112-
"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage",
113-
},
114-
}
115-
{%- endif %}
116-
{%- endmacro -%}
117-
118-
{#- ==========================================
119-
Render according to test_storages_compat_layer
120-
==============================================
121-
122-
Case 1) test_storages_compat_layer is unset
123-
- All storages render the legacy setting
124-
- Branches using pulpcore <3.70 must leave this key unset (use legacy)
125-
126-
Case 2) test_storages_compat_layer is True
127-
- To tests both work, only one setting uses the new storage setting
128-
- Branches using pulpcore >=3.70,<3.85 must set this key to True (test both)
129-
130-
Case 3) test_storages_compat_layer is False
131-
- All storages render the new setting
132-
- Branches using pulpcore >=3.85 must set this key to False (use new)
133-
-#}
134-
135-
{% if s3_test | default(false) or azure_test | default(false) or gcp_test | default(false)%}
136-
MEDIA_ROOT=""
137-
{% endif %}
138-
{%- if test_storages_compat_layer is not defined -%}
139-
{%- if s3_test | default(false) -%}{{ s3_settings(legacy=True) }}{%- endif -%}
140-
{%- if azure_test | default(false) -%}{{ azure_settings(legacy=True) }}{%- endif -%}
141-
{%- if gcp_test | default(false) -%}{{ gcp_settings(legacy=True) }}{%- endif -%}
142-
{%- else -%}
143-
{%- if test_storages_compat_layer is true -%}
144-
{%- if s3_test | default(false) -%}{{ s3_settings(legacy=False) }}{%- endif -%}
145-
{%- if azure_test | default(false) -%}{{ azure_settings(legacy=True) }}{%- endif -%}
146-
{%- if gcp_test | default(false) -%}{{ gcp_settings(legacy=True) }}{%- endif -%}
147-
{%- elif test_storages_compat_layer is false -%}
148-
{%- if s3_test | default(false) -%}{{ s3_settings(legacy=False) }}{%- endif -%}
149-
{%- if azure_test | default(false) -%}{{ azure_settings(legacy=False) }}{%- endif -%}
150-
{%- if gcp_test | default(false) -%}{{ gcp_settings(legacy=False) }}{%- endif -%}
151-
{%- endif -%}
152-
{%- endif -%}

‎.github/workflows/scripts/install.sh‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -95,9 +95,8 @@ if [ "$TEST" = "s3" ]; then
9595
sed -i -e '$a s3_test: true\
9696
minio_access_key: "'$MINIO_ACCESS_KEY'"\
9797
minio_secret_key: "'$MINIO_SECRET_KEY'"\
98-
pulp_scenario_settings: {"domain_enabled": true, "flatpak_index": false, "token_auth_disabled": true}\
98+
pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.s3boto3.S3Boto3Storage", "OPTIONS": {"access_key": "AKIAIT2Z5TDYPX3ARJBA", "addressing_style": "path", "bucket_name": "pulp3", "default_acl": "@none", "endpoint_url": "http://minio:9000", "region_name": "eu-central-1", "secret_key": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "signature_version": "s3v4"}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "domain_enabled": true, "flatpak_index": false, "token_auth_disabled": true}\
9999
pulp_scenario_env: {}\
100-
test_storages_compat_layer: false\
101100
' vars/main.yaml
102101
export PULP_API_ROOT="/rerouted/djnd/"
103102
fi
@@ -110,9 +109,8 @@ if [ "$TEST" = "azure" ]; then
110109
- ./azurite:/etc/pulp\
111110
command: "azurite-blob --blobHost 0.0.0.0"' vars/main.yaml
112111
sed -i -e '$a azure_test: true\
113-
pulp_scenario_settings: {"content_origin": null, "domain_enabled": true, "flatpak_index": true}\
112+
pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.azure_storage.AzureStorage", "OPTIONS": {"account_key": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", "account_name": "devstoreaccount1", "azure_container": "pulp-test", "connection_string": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;", "expiration_secs": 120, "location": "pulp3", "overwrite_files": true}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "content_origin": null, "domain_enabled": true, "flatpak_index": true}\
114113
pulp_scenario_env: {}\
115-
test_storages_compat_layer: false\
116114
' vars/main.yaml
117115
fi
118116

‎pulp_container/tests/functional/api/test_sync.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ def test_basic_sync(
5959

6060
latest_version_href = repository.latest_version_href
6161
media_type = MEDIA_TYPE.MANIFEST_V2
62-
if has_pulp_plugin("core", min="3.70"):
62+
if has_pulp_plugin("core", min="3.70", max="3.85"):
6363
media_type = quote(media_type)
6464

6565
assert check_manifest_fields(

‎template_config.yml‎

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,11 +54,40 @@ pulp_settings:
5454
- /tmp
5555
flatpak_index: true
5656
pulp_settings_azure:
57+
MEDIA_ROOT: ''
58+
STORAGES:
59+
default:
60+
BACKEND: storages.backends.azure_storage.AzureStorage
61+
OPTIONS:
62+
account_key: Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
63+
account_name: devstoreaccount1
64+
azure_container: pulp-test
65+
connection_string: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;
66+
expiration_secs: 120
67+
location: pulp3
68+
overwrite_files: true
69+
staticfiles:
70+
BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage
5771
content_origin: null
5872
domain_enabled: true
5973
flatpak_index: true
6074
pulp_settings_gcp: null
6175
pulp_settings_s3:
76+
MEDIA_ROOT: ''
77+
STORAGES:
78+
default:
79+
BACKEND: storages.backends.s3boto3.S3Boto3Storage
80+
OPTIONS:
81+
access_key: AKIAIT2Z5TDYPX3ARJBA
82+
addressing_style: path
83+
bucket_name: pulp3
84+
default_acl: '@none'
85+
endpoint_url: http://minio:9000
86+
region_name: eu-central-1
87+
secret_key: fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS
88+
signature_version: s3v4
89+
staticfiles:
90+
BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage
6291
domain_enabled: true
6392
flatpak_index: false
6493
token_auth_disabled: true
@@ -84,6 +113,5 @@ test_lowerbounds: true
84113
test_performance: false
85114
test_reroute: true
86115
test_s3: true
87-
test_storages_compat_layer: false
88116
use_issue_template: true
89117

0 commit comments

Comments
 (0)