diff --git a/.github/workflows/ceph-ns-aws-s3-tests.yaml b/.github/workflows/ceph-ns-aws-s3-tests.yaml new file mode 100644 index 0000000000..33f3194b58 --- /dev/null +++ b/.github/workflows/ceph-ns-aws-s3-tests.yaml @@ -0,0 +1,34 @@ +name: Ceph NS AWS S3 Tests +on: [workflow_call] + +jobs: + ceph-ns-aws-s3-tests: + name: Ceph NS AWS S3 Tests + runs-on: ubuntu-latest + timeout-minutes: 90 + permissions: + actions: read # download-artifact + contents: read # required for actions/checkout + steps: + - name: Checkout noobaa-core + uses: actions/checkout@v4 + with: + repository: 'noobaa/noobaa-core' + path: 'noobaa-core' + + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: noobaa-tester + path: /tmp + + - name: Load image + run: docker load --input /tmp/noobaa-tester.tar + + - name: Run Ceph NS AWS s3-tests + run: | + set -x + cd ./noobaa-core + mkdir -p logs/ceph-ns-aws-test-logs + chmod 777 logs/ceph-ns-aws-test-logs + make test-cephs3-ns-aws -o tester diff --git a/.github/workflows/run-pr-tests.yaml b/.github/workflows/run-pr-tests.yaml index 75d4054911..87909fb10d 100644 --- a/.github/workflows/run-pr-tests.yaml +++ b/.github/workflows/run-pr-tests.yaml @@ -32,6 +32,10 @@ jobs: needs: build-noobaa-image uses: ./.github/workflows/ceph-s3-tests.yaml + ceph-ns-aws-s3-tests: + needs: build-noobaa-image + uses: ./.github/workflows/ceph-ns-aws-s3-tests.yaml + ceph-nsfs-s3-tests: needs: build-noobaa-image uses: ./.github/workflows/ceph-nsfs-s3-tests.yaml diff --git a/Makefile b/Makefile index 289fe9e1f1..5976e990a0 100644 --- a/Makefile +++ b/Makefile @@ -111,6 +111,17 @@ MINT_MOCK_ACCESS_KEY="aaaaaaaaaaaaaEXAMPLE" MINT_MOCK_SECRET_KEY="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaEXAMPLE" MINT_NOOBAA_HTTP_ENDPOINT_PORT=6001 +#################### +# MINIO VARIABLES # +################### + +MINIO_IMAGE?="quay.io/minio/minio" + +MINIO_PORT?=9000 +MINIO_USER?="miniotest" +MINIO_PASSWORD?="miniotest123" +MINIO_TEST_BUCKET?="minio-test-bucket" + ############### # BUILD LOCAL # ############### @@ -352,6 +363,19 @@ test-cephs3: tester @$(call remove_docker_network) .PHONY: test-cephs3 +test-cephs3-ns-aws: tester + @echo "\033[1;34mRunning ceph-s3 tests with namespace store pointing to s3 bucket in Minio\033[0m" + @$(call create_docker_network) + @$(call run_minio) + @$(call run_postgres) + @echo "\033[1;34mRunning tests\033[0m" + $(CONTAINER_ENGINE) run $(CPUSET) --privileged --user root --network noobaa-net --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" --env "POSTGRES_HOST=coretest-postgres-$(GIT_COMMIT)-$(NAME_POSTFIX)" --env "POSTGRES_USER=noobaa" --env "DB_TYPE=postgres" --env "POSTGRES_DBNAME=coretest" --env "USE_S3_NAMESPACE_RESOURCE=true" --env "MINIO_ENDPOINT=http://coretest-minio-$(GIT_COMMIT)-$(NAME_POSTFIX):$(MINIO_PORT)" --env "MINIO_USER=$(MINIO_USER)" --env "MINIO_PASSWORD=$(MINIO_PASSWORD)" --env "MINIO_TEST_BUCKET=$(MINIO_TEST_BUCKET)" -v $(PWD)/logs:/logs $(TESTER_TAG) "./src/test/external_tests/ceph_s3_tests/run_ceph_test_on_test_container.sh" + @$(call stop_noobaa) + @$(call stop_postgres) + @$(call stop_minio) + @$(call remove_docker_network) +.PHONY: test-cephs3-ns-aws + test-warp: tester @echo "\033[1;34mRunning warp tests with Postgres.\033[0m" @$(call create_docker_network) @@ -581,3 +605,23 @@ define stop_blob_mock fi @echo "\033[1;32mBlob mock server stop done.\033[0m" endef + +######### +# MINIO # +######### + +define run_minio + @echo "\033[1;34mRunning Minio container\033[0m" + $(CONTAINER_ENGINE) run -d $(CPUSET) --network noobaa-net --name coretest-minio-$(GIT_COMMIT)-$(NAME_POSTFIX) -p $(MINIO_PORT):$(MINIO_PORT) --env "MINIO_ROOT_USER=$(MINIO_USER)" --env "MINIO_ROOT_PASSWORD=$(MINIO_PASSWORD)" $(MINIO_IMAGE) server /data --address ":$(MINIO_PORT)" + @echo "\033[1;34mWaiting for Minio to start..\033[0m" + sleep 20 + @echo "\033[1;32mRun Minio done.\033[0m" +endef + +define stop_minio + @echo "\033[1;34mStopping/removing Minio container\033[0m" + $(call disconnect_container_from_noobaa_network, coretest-minio-$(GIT_COMMIT)-$(NAME_POSTFIX)) + $(CONTAINER_ENGINE) stop coretest-minio-$(GIT_COMMIT)-$(NAME_POSTFIX) + $(CONTAINER_ENGINE) rm coretest-minio-$(GIT_COMMIT)-$(NAME_POSTFIX) + @echo "\033[1;32mStop Minio done.\033[0m" +endef diff --git a/src/test/external_tests/ceph_s3_tests/run_ceph_test_on_test_container.sh b/src/test/external_tests/ceph_s3_tests/run_ceph_test_on_test_container.sh index 1b1ef28b81..8a343cbc1f 100755 --- a/src/test/external_tests/ceph_s3_tests/run_ceph_test_on_test_container.sh +++ b/src/test/external_tests/ceph_s3_tests/run_ceph_test_on_test_container.sh @@ -34,7 +34,13 @@ export MGMT_ADDR=wss://${NOOBAA_MGMT_SERVICE_HOST:-localhost}:${NOOBAA_MGMT_SERV export BG_ADDR=wss://localhost:5445 export HOSTED_AGENTS_ADDR=wss://localhost:5446 -export CEPH_TEST_LOGS_DIR=/logs/ceph-test-logs +# set logs directory based on test type +# later we will add more test types and logs directories on the basis of s3 namespace resource type +if [ "${USE_S3_NAMESPACE_RESOURCE}" = "true" ]; then + export CEPH_TEST_LOGS_DIR=/logs/ceph-ns-aws-test-logs +else + export CEPH_TEST_LOGS_DIR=/logs/ceph-test-logs +fi export CONFIG_JS_OBJECT_SDK_BUCKET_CACHE_EXPIRY_MS=0 # Needed for disabling cache for ceph cors test and maybe some more export CONFIG_JS_allow_anonymous_access_in_test=true # Needed for allowing anon access for tests using ACL='public-read-write' diff --git a/src/test/external_tests/ceph_s3_tests/s3-tests-lists/ns_aws_s3_tests_black_list.txt b/src/test/external_tests/ceph_s3_tests/s3-tests-lists/ns_aws_s3_tests_black_list.txt new file mode 100644 index 0000000000..d54490a7ce --- /dev/null +++ b/src/test/external_tests/ceph_s3_tests/s3-tests-lists/ns_aws_s3_tests_black_list.txt @@ -0,0 +1,437 @@ +s3tests/functional/test_s3.py::test_100_continue +s3tests/functional/test_headers.py::test_object_create_date_and_amz_date +s3tests/functional/test_headers.py::test_object_create_amz_date_and_no_date +s3tests/functional/test_headers.py::test_object_create_bad_authorization_empty +s3tests/functional/test_headers.py::test_object_create_bad_contentlength_none +s3tests/functional/test_headers.py::test_object_create_bad_contentlength_mismatch_below_aws2 +s3tests/functional/test_s3.py::test_versioning_obj_read_not_exist_null +s3tests/functional/test_s3.py::test_append_normal_object +s3tests/functional/test_s3.py::test_append_object +s3tests/functional/test_s3.py::test_append_object_position_wrong +s3tests/functional/test_s3.py::test_logging_toggle +s3tests/functional/test_s3.py::test_encryption_sse_c_multipart_invalid_chunks_2 +s3tests/functional/test_s3.py::test_bucket_policy_different_tenant +s3tests/functional/test_s3.py::test_bucket_policy_set_condition_operator_end_with_IfExists +s3tests/functional/test_s3.py::test_bucket_policy_put_obj_enc +s3tests/functional/test_s3.py::test_bucket_policy_put_obj_request_obj_tag +s3tests/functional/test_s3_website.py::check_can_test_website +s3tests/functional/test_s3_website.py::test_website_nonexistant_bucket_s3 +s3tests/functional/test_s3_website.py::test_website_nonexistant_bucket_rgw +s3tests/functional/test_s3_website.py::test_website_public_bucket_list_public_index +s3tests/functional/test_s3_website.py::test_website_private_bucket_list_public_index +s3tests/functional/test_s3_website.py::test_website_public_bucket_list_empty +s3tests/functional/test_s3_website.py::test_website_private_bucket_list_empty +s3tests/functional/test_s3_website.py::test_website_private_bucket_list_private_index +s3tests/functional/test_s3_website.py::test_website_private_bucket_list_empty_missingerrordoc +s3tests/functional/test_s3_website.py::test_website_public_bucket_list_private_index +s3tests/functional/test_s3_website.py::test_website_public_bucket_list_private_index_missingerrordoc +s3tests/functional/test_s3_website.py::test_website_private_bucket_list_private_index_missingerrordoc +s3tests/functional/test_s3_website.py::test_website_public_bucket_list_empty_missingerrordoc +s3tests/functional/test_s3_website.py::test_website_private_bucket_list_empty_blockederrordoc +s3tests/functional/test_s3_website.py::test_website_public_bucket_list_empty_blockederrordoc +s3tests/functional/test_s3_website.py::test_website_public_bucket_list_private_index_blockederrordoc +s3tests/functional/test_s3_website.py::test_website_private_bucket_list_private_index_blockederrordoc +s3tests/functional/test_s3_website.py::test_website_private_bucket_list_empty_gooderrordoc +s3tests/functional/test_s3_website.py::test_website_public_bucket_list_empty_gooderrordoc +s3tests/functional/test_s3_website.py::test_website_public_bucket_list_private_index_gooderrordoc +s3tests/functional/test_s3_website.py::test_website_private_bucket_list_private_index_gooderrordoc +s3tests/functional/test_s3_website.py::test_website_bucket_private_redirectall_base +s3tests/functional/test_s3_website.py::test_website_bucket_private_redirectall_path +s3tests/functional/test_s3_website.py::test_website_bucket_private_redirectall_path_upgrade +s3tests/functional/test_s3_website.py::test_website_xredirect_nonwebsite +s3tests/functional/test_s3_website.py::test_website_xredirect_private_relative +s3tests/functional/test_s3_website.py::test_website_xredirect_public_abs +s3tests/functional/test_s3_website.py::test_website_xredirect_public_relative +s3tests/functional/test_s3_website.py +s3tests/functional/test_s3_website.py::test_website_xredirect_private_abs +test_realistic/TestFileValidator.py::test_new_file_is_valid_on_several_calls +test_realistic/TestFileValidator.py::test_new_file_is_valid_when_size_is_1 +test_realistic/TestFileValidator.py::test_new_file_is_valid +test_realistic/TestFiles/test_random_file_valid +s3tests_boto3/functional/test_headers.py::test_object_create_bad_contentlength_none +s3tests_boto3/functional/test_headers.py::test_object_create_bad_contentlength_mismatch_abov +s3tests_boto3/functional/test_headers.py::test_object_create_bad_authorization_empty +s3tests_boto3/functional/test_headers.py::test_object_create_date_and_amz_date +s3tests_boto3/functional/test_headers.py::test_object_create_amz_date_and_no_date +s3tests_boto3/functional/test_headers.py::test_object_create_bad_authorization_none +s3tests_boto3/functional/test_headers.py::test_bucket_put_bad_canned_acl +s3tests_boto3/functional/test_headers.py::test_bucket_create_bad_expect_mismatch +s3tests_boto3/functional/test_headers.py::test_bucket_create_bad_authorization_none +s3tests_boto3/functional/test_headers.py::test_bucket_create_bad_authorization_empty +s3tests_boto3/functional/test_headers.py::test_object_create_bad_contentlength_mismatch_below_aws2 +s3tests_boto3/functional/test_headers.py::test_object_create_bad_authorization_incorrect_aws2 +s3tests_boto3/functional/test_headers.py::test_object_create_bad_authorization_invalid_aws2 +s3tests_boto3/functional/test_headers.py::test_object_create_bad_date_invalid_aws2 +s3tests_boto3/functional/test_headers.py::test_object_create_bad_date_empty_aws2 +s3tests_boto3/functional/test_headers.py::test_object_create_bad_date_none_aws2 +s3tests_boto3/functional/test_headers.py::test_object_create_bad_date_before_epoch_aws2 +s3tests_boto3/functional/test_headers.py::test_bucket_create_bad_authorization_invalid_aws2 +s3tests_boto3/functional/test_headers.py::test_bucket_create_bad_date_invalid_aws2 +s3tests_boto3/functional/test_headers.py::test_bucket_create_bad_date_none_aws2 +s3tests_boto3/functional/test_headers.py::test_bucket_create_bad_date_empty_aws2 +s3tests_boto3/functional/test_headers.py::test_bucket_create_bad_date_before_epoch_aws2 +s3tests/functional/test_headers.py::test_object_acl_create_contentlength_none +s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_prefix +s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_prefix_underscore +s3tests_boto3/functional/test_s3.py::test_bucket_list_delimiter_unreadable +s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_unreadable +s3tests_boto3/functional/test_s3.py::test_bucket_listv2_fetchowner_empty +s3tests/functional/test_s3.py::test_encryption_sse_c_multipart_invalid_chunks_1 +s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_unreadable +s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_unreadable +s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_delimiter_not_exist +s3tests_boto3/functional/test_s3.py::test_bucket_listv2_maxkeys_zero +s3tests_boto3/functional/test_s3.py::test_bucket_list_unordered +s3tests_boto3/functional/test_s3.py::test_bucket_listv2_unordered +s3tests_boto3/functional/test_s3.py::test_bucket_listv2_continuationtoken +s3tests_boto3/functional/test_s3.py::test_bucket_listv2_startafter_unreadable +s3tests_boto3/functional/test_s3.py::test_bucket_list_return_data_versioning +s3tests_boto3/functional/test_s3.py::test_bucket_list_objects_anonymous +s3tests_boto3/functional/test_s3.py::test_bucket_listv2_objects_anonymous +s3tests_boto3/functional/test_s3.py::test_object_write_cache_control +s3tests_boto3/functional/test_s3.py::test_object_write_expires +s3tests_boto3/functional/test_s3.py::test_multi_objectv2_delete +s3tests_boto3/functional/test_s3.py::test_object_set_get_unicode_metadata +s3tests_boto3/functional/test_s3.py::test_object_set_get_metadata_empty_to_unreadable_prefix +s3tests_boto3/functional/test_s3.py::test_object_set_get_non_utf8_metadata +s3tests_boto3/functional/test_s3.py::test_object_set_get_metadata_empty_to_unreadable_suffix +s3tests_boto3/functional/test_s3.py::test_object_set_get_metadata_empty_to_unreadable_infix +s3tests_boto3/functional/test_s3.py::test_object_set_get_metadata_overwrite_to_unreadable_prefix +s3tests_boto3/functional/test_s3.py::test_object_set_get_metadata_overwrite_to_unreadable_suffix +s3tests_boto3/functional/test_s3.py::test_object_set_get_metadata_overwrite_to_unreadable_infix +s3tests_boto3/functional/test_s3.py::test_post_object_anonymous_request +s3tests_boto3/functional/test_s3.py::test_post_object_authenticated_request +s3tests_boto3/functional/test_s3.py::test_post_object_authenticated_no_content_type +s3tests_boto3/functional/test_s3.py::test_post_object_set_success_code +s3tests_boto3/functional/test_s3.py::test_post_object_set_invalid_success_code +s3tests_boto3/functional/test_s3.py::test_post_object_upload_larger_than_chunk +s3tests_boto3/functional/test_s3.py::test_post_object_set_key_from_filename +s3tests_boto3/functional/test_s3.py::test_post_object_case_insensitive_condition_fields +s3tests_boto3/functional/test_s3.py::test_post_object_ignored_header +s3tests_boto3/functional/test_s3.py::test_post_object_escaped_field_values +s3tests_boto3/functional/test_s3.py::test_post_object_success_redirect_action +s3tests_boto3/functional/test_s3.py::test_post_object_invalid_date_format +s3tests_boto3/functional/test_s3.py::test_post_object_no_key_specified +s3tests_boto3/functional/test_s3.py::test_post_object_missing_signature +s3tests_boto3/functional/test_s3.py::test_post_object_user_specified_header +s3tests_boto3/functional/test_s3.py::test_post_object_condition_is_case_sensitive +s3tests_boto3/functional/test_s3.py::test_post_object_expires_is_case_sensitive +s3tests_boto3/functional/test_s3.py::test_post_object_missing_expires_condition +s3tests_boto3/functional/test_s3.py::test_post_object_missing_conditions_list +s3tests_boto3/functional/test_s3.py::test_post_object_upload_size_limit_exceeded +s3tests_boto3/functional/test_s3.py::test_post_object_missing_content_length_argument +s3tests_boto3/functional/test_s3.py::test_post_object_invalid_content_length_argument +s3tests_boto3/functional/test_s3.py::test_post_object_upload_size_below_minimum +s3tests_boto3/functional/test_s3.py::test_post_object_empty_conditions +s3tests_boto3/functional/test_s3.py::test_object_raw_get_bucket_gone +s3tests_boto3/functional/test_s3.py::test_object_raw_get +s3tests_boto3/functional/test_s3.py::test_object_delete_key_bucket_gone +s3tests_boto3/functional/test_s3.py::test_object_raw_get_object_gone +s3tests_boto3/functional/test_s3.py::test_bucket_head_extended +s3tests_boto3/functional/test_s3.py::test_object_raw_get_bucket_acl +s3tests_boto3/functional/test_s3.py::test_object_raw_get_object_acl +s3tests_boto3/functional/test_s3.py::test_object_raw_response_headers +s3tests_boto3/functional/test_s3.py::test_object_raw_authenticated_bucket_acl +s3tests_boto3/functional/test_s3.py::test_object_raw_get_x_amz_expires_out_range_zero +s3tests_boto3/functional/test_s3.py::test_object_raw_get_x_amz_expires_out_positive_range +s3tests_boto3/functional/test_s3.py::test_object_raw_get_x_amz_expires_out_max_range +s3tests_boto3/functional/test_s3.py::test_object_anon_put_write_access +s3tests_boto3/functional/test_s3.py::test_object_raw_put_authenticated_expired +s3tests_boto3/functional/test_s3.py::test_bucket_create_naming_bad_short_empty +s3tests_boto3/functional/test_s3.py::test_bucket_create_naming_good_long_250 +s3tests_boto3/functional/test_s3.py::test_bucket_create_naming_good_long_251 +s3tests_boto3/functional/test_s3.py::test_bucket_create_naming_good_long_252 +s3tests_boto3/functional/test_s3.py::test_bucket_create_naming_good_long_253 +s3tests_boto3/functional/test_s3.py::test_bucket_create_naming_good_long_254 +s3tests_boto3/functional/test_s3.py::test_bucket_create_naming_good_long_255 +s3tests_boto3/functional/test_s3.py::test_bucket_list_long_name +s3tests_boto3/functional/test_s3.py::test_bucket_create_naming_dns_underscore +s3tests_boto3/functional/test_s3.py::test_bucket_create_naming_dns_long +s3tests_boto3/functional/test_s3.py::test_bucket_create_naming_dns_dash_at_end +s3tests_boto3/functional/test_s3.py::test_bucket_create_naming_dns_dot_dot +s3tests_boto3/functional/test_s3.py::test_bucket_create_naming_dns_dot_dash +s3tests_boto3/functional/test_s3.py::test_bucket_create_naming_dns_dash_dot +s3tests_boto3/functional/test_s3.py::test_bucket_acl_default +s3tests_boto3/functional/test_s3.py::test_bucket_acl_canned_during_create +s3tests_boto3/functional/test_s3.py::test_bucket_acl_canned +s3tests_boto3/functional/test_s3.py::test_bucket_acl_canned_publicreadwrite +s3tests_boto3/functional/test_s3.py::test_bucket_acl_canned_authenticatedread +s3tests_boto3/functional/test_s3.py::test_object_acl_default +s3tests_boto3/functional/test_s3.py::test_object_acl_canned_during_create +s3tests_boto3/functional/test_s3.py::test_object_acl_canned_publicreadwrite +s3tests_boto3/functional/test_s3.py::test_object_acl_canned_bucketownerread +s3tests_boto3/functional/test_s3.py::test_object_acl_canned_authenticatedread +s3tests_boto3/functional/test_s3.py::test_object_acl_canned_bucketownerfullcontrol +s3tests_boto3/functional/test_s3.py::test_object_acl_full_control_verify_owner +s3tests_boto3/functional/test_s3.py::test_object_acl_full_control_verify_attributes +s3tests_boto3/functional/test_s3.py::test_object_acl +s3tests_boto3/functional/test_s3.py::test_object_acl_write +s3tests_boto3/functional/test_s3.py::test_object_acl_writeacp +s3tests_boto3/functional/test_s3.py::test_object_acl_read +s3tests_boto3/functional/test_s3.py::test_object_acl_readacp +s3tests_boto3/functional/test_s3.py::test_bucket_acl_grant_userid_fullcontrol +s3tests_boto3/functional/test_s3.py::test_bucket_acl_grant_userid_read +s3tests_boto3/functional/test_s3.py::test_bucket_acl_grant_userid_readacp +s3tests_boto3/functional/test_s3.py::test_bucket_acl_grant_userid_write +s3tests_boto3/functional/test_s3.py::test_bucket_acl_grant_userid_writeacp +s3tests_boto3/functional/test_s3.py::test_bucket_acl_grant_nonexist_user +s3tests_boto3/functional/test_s3.py::test_bucket_acl_no_grants +s3tests_boto3/functional/test_s3.py::test_bucket_header_acl_grants +s3tests_boto3/functional/test_s3.py::test_object_header_acl_grants +s3tests_boto3/functional/test_s3.py::test_bucket_acl_grant_email +s3tests_boto3/functional/test_s3.py::test_bucket_acl_grant_email_not_exist +s3tests_boto3/functional/test_s3.py::test_bucket_acl_revoke_all +s3tests_boto3/functional/test_s3.py::test_logging_toggle +s3tests_boto3/functional/test_s3.py::test_access_bucket_private_object_private +s3tests_boto3/functional/test_s3.py::test_access_bucket_private_objectv2_private +s3tests_boto3/functional/test_s3.py::test_access_bucket_private_object_publicread +s3tests_boto3/functional/test_s3.py::test_object_acl_canned +s3tests_boto3/functional/test_s3.py::test_access_bucket_private_objectv2_publicread +s3tests_boto3/functional/test_s3.py::test_access_bucket_private_object_publicreadwrite +s3tests_boto3/functional/test_s3.py::test_access_bucket_private_objectv2_publicreadwrite +s3tests_boto3/functional/test_s3.py::test_access_bucket_publicread_object_private +s3tests_boto3/functional/test_s3.py::test_access_bucket_publicread_object_publicread +s3tests_boto3/functional/test_s3.py::test_access_bucket_publicread_object_publicreadwrite +s3tests_boto3/functional/test_s3.py::test_access_bucket_publicreadwrite_object_private +s3tests_boto3/functional/test_s3.py::test_access_bucket_publicreadwrite_object_publicread +s3tests_boto3/functional/test_s3.py::test_list_buckets_anonymous +s3tests_boto3/functional/test_s3.py::test_access_bucket_publicreadwrite_object_publicreadwrite +s3tests_boto3/functional/test_s3.py::test_list_buckets_invalid_auth +s3tests_boto3/functional/test_s3.py::test_list_buckets_bad_auth +s3tests_boto3/functional/test_s3.py::test_bucket_recreate_not_overriding +s3tests_boto3/functional/test_s3.py::test_object_copy_to_itself +s3tests_boto3/functional/test_s3.py::test_object_copy_not_owned_bucket +s3tests_boto3/functional/test_s3.py::test_object_copy_not_owned_object_bucket +s3tests_boto3/functional/test_s3.py::test_multipart_copy_invalid_range +s3tests_boto3/functional/test_s3.py::test_multipart_upload +s3tests_boto3/functional/test_s3.py::test_multipart_upload_size_too_small +s3tests_boto3/functional/test_s3.py::test_abort_multipart_upload +s3tests_boto3/functional/test_s3.py::test_multipart_copy_improper_range +s3tests_boto3/functional/test_s3.py::test_100_continue +s3tests_boto3/functional/test_s3.py::test_set_tagging +s3tests_boto3/functional/test_s3.py::test_multipart_resend_first_finishes_last +s3tests_boto3/functional/test_s3.py::test_versioned_object_acl +s3tests_boto3/functional/test_s3.py::test_versioned_object_acl_no_version_specified +s3tests_boto3/functional/test_s3.py::test_lifecycle_get +s3tests_boto3/functional/test_s3.py::test_lifecycle_id_too_long +s3tests_boto3/functional/test_s3.py::test_lifecycle_same_id +s3tests_boto3/functional/test_s3.py::test_lifecycle_invalid_status +s3tests_boto3/functional/test_s3.py::test_lifecycle_set_invalid_date +s3tests_boto3/functional/test_s3.py::test_lifecycle_expiration_date +s3tests_boto3/functional/test_s3.py::test_lifecycle_expiration_days0 +s3tests_boto3/functional/test_s3.py::test_lifecycle_set_noncurrent +s3tests_boto3/functional/test_s3.py::test_lifecycle_noncur_expiration +s3tests_boto3/functional/test_s3.py::test_lifecycle_set_deletemarker +s3tests_boto3/functional/test_s3.py::test_lifecycle_set_filter +s3tests_boto3/functional/test_s3.py::test_lifecycle_deletemarker_expiration +s3tests_boto3/functional/test_s3.py::test_lifecycle_set_empty_filter +s3tests_boto3/functional/test_s3.py::test_lifecyclev2_expiration +s3tests_boto3/functional/test_s3.py::test_lifecycle_set_multipart +s3tests_boto3/functional/test_s3.py::test_encryption_sse_c_multipart_upload +s3tests_boto3/functional/test_s3.py::test_encryption_sse_c_post_object_authenticated_request +s3tests_boto3/functional/test_s3.py::test_lifecycle_multipart_expiration +s3tests_boto3/functional/test_s3.py::test_encryption_sse_c_multipart_bad_download +s3tests_boto3/functional/test_s3.py::test_sse_kms_multipart_upload +s3tests_boto3/functional/test_s3.py::test_sse_kms_post_object_authenticated_request +s3tests_boto3/functional/test_s3.py::test_bucket_policy_acl +s3tests_boto3/functional/test_s3.py::test_bucketv2_policy_acl +s3tests_boto3/functional/test_s3.py::test_bucket_policy_different_tenant +s3tests_boto3/functional/test_s3.py::test_encryption_sse_c_multipart_invalid_chunks_1 +s3tests_boto3/functional/test_s3.py::test_bucketv2_policy_different_tenant +s3tests_boto3/functional/test_s3.py::test_bucket_policy_set_condition_operator_end_with_IfExists +s3tests_boto3/functional/test_s3.py::test_post_object_tags_anonymous_request +s3tests_boto3/functional/test_s3.py::test_post_object_tags_authenticated_request +s3tests_boto3/functional/test_s3.py::test_put_obj_with_tags +s3tests_boto3/functional/test_s3.py::test_get_tags_acl_public +s3tests_boto3/functional/test_s3.py::test_delete_tags_obj_public +s3tests_boto3/functional/test_s3.py::test_put_tags_acl_public +s3tests_boto3/functional/test_s3.py::test_bucket_policy_get_obj_tagging_existing_tag +s3tests_boto3/functional/test_s3.py::test_bucket_policy_get_obj_existing_tag +s3tests_boto3/functional/test_s3.py::test_bucket_policy_put_obj_tagging_existing_tag +s3tests_boto3/functional/test_s3.py::test_bucket_policy_put_obj_copy_source +s3tests_boto3/functional/test_s3.py::test_bucket_policy_put_obj_copy_source_meta +s3tests_boto3/functional/test_s3.py::test_bucket_policy_put_obj_acl +s3tests_boto3/functional/test_s3.py::test_bucket_policy_put_obj_enc +s3tests_boto3/functional/test_s3.py::test_bucket_policy_put_obj_grant +s3tests_boto3/functional/test_s3.py::test_bucket_policy_put_obj_request_obj_tag +s3tests_boto3/functional/test_s3.py::test_object_lock_put_obj_lock +s3tests_boto3/functional/test_s3.py::test_object_lock_put_obj_lock_invalid_bucket +s3tests_boto3/functional/test_s3.py::test_bucket_policy_get_obj_acl_existing_tag +s3tests_boto3/functional/test_s3.py::test_object_lock_put_obj_lock_with_days_and_years +s3tests_boto3/functional/test_s3.py::test_object_lock_put_obj_lock_invalid_days +s3tests_boto3/functional/test_s3.py::test_object_lock_put_obj_lock_invalid_years +s3tests_boto3/functional/test_s3.py::test_object_lock_put_obj_lock_invalid_status +s3tests_boto3/functional/test_s3.py::test_object_lock_suspend_versioning +s3tests_boto3/functional/test_s3.py::test_object_lock_get_obj_lock +s3tests_boto3/functional/test_s3.py::test_object_lock_get_obj_lock_invalid_bucket +s3tests_boto3/functional/test_s3.py::test_object_lock_put_obj_retention +s3tests_boto3/functional/test_s3.py::test_object_lock_put_obj_retention_invalid_bucket +s3tests_boto3/functional/test_s3.py::test_object_lock_put_obj_retention_invalid_mode +s3tests_boto3/functional/test_s3.py::test_object_lock_get_obj_retention +s3tests_boto3/functional/test_s3.py::test_object_lock_get_obj_retention_invalid_bucket +s3tests_boto3/functional/test_s3.py::test_object_lock_put_obj_retention_versionid +s3tests_boto3/functional/test_s3.py::test_object_lock_put_obj_retention_override_default_retention +s3tests_boto3/functional/test_s3.py::test_object_lock_put_obj_retention_increase_period +s3tests_boto3/functional/test_s3.py::test_object_lock_put_obj_retention_shorten_period +s3tests_boto3/functional/test_s3.py::test_object_lock_put_obj_retention_shorten_period_bypass +s3tests_boto3/functional/test_s3.py::test_object_lock_delete_object_with_retention +s3tests_boto3/functional/test_s3.py::test_object_lock_put_legal_hold_invalid_bucket +s3tests_boto3/functional/test_s3.py::test_object_lock_put_legal_hold_invalid_status +s3tests_boto3/functional/test_s3.py::test_object_lock_get_legal_hold_invalid_bucket +s3tests_boto3/functional/test_s3.py::test_object_lock_delete_object_with_legal_hold_on +s3tests_boto3/functional/test_s3.py::test_object_lock_delete_object_with_legal_hold_off +s3tests_boto3/functional/test_s3.py::test_object_lock_get_obj_metadata +s3tests_boto3/functional/test_s3.py::test_object_lock_uploading_obj +s3tests_boto3/fuzz/test/test_fuzzer/test_load_graph +test_realistic/TestFileValidator.py::test_new_file_is_valid +test_realistic/TestFileValidator.py::test_new_file_is_valid_on_several_calls +test_realistic/TestFileValidator.py::test_new_file_is_valid_when_size_is_1 +test_realistic/TestFiles/test_random_file_valid +s3tests/functional/test_headers.py::test_object_create_bad_contentlength_mismatch_above +s3tests_boto3/functional/test_iam.py::test_put_user_policy +s3tests_boto3/functional/test_iam.py::test_put_user_policy_invalid_user +s3tests_boto3/functional/test_iam.py::test_put_user_policy_parameter_limit +s3tests_boto3/functional/test_iam.py::test_put_user_policy_invalid_element +s3tests_boto3/functional/test_iam.py::test_put_existing_user_policy +s3tests_boto3/functional/test_iam.py::test_list_user_policy +s3tests_boto3/functional/test_iam.py::test_list_user_policy_invalid_user +s3tests_boto3/functional/test_iam.py::test_get_user_policy +s3tests_boto3/functional/test_iam.py::test_get_user_policy_invalid_user +s3tests_boto3/functional/test_iam.py::test_get_user_policy_invalid_policy_name +s3tests_boto3/functional/test_iam.py::test_get_deleted_user_policy +s3tests_boto3/functional/test_iam.py::test_get_user_policy_from_multiple_policies +s3tests_boto3/functional/test_iam.py::test_delete_user_policy +s3tests_boto3/functional/test_iam.py::test_delete_user_policy_invalid_user +s3tests_boto3/functional/test_iam.py::test_delete_user_policy_invalid_policy_name +s3tests_boto3/functional/test_iam.py::test_delete_user_policy_from_multiple_policies +s3tests_boto3/functional/test_iam.py::test_allow_bucket_actions_in_user_policy +s3tests_boto3/functional/test_iam.py::test_deny_bucket_actions_in_user_policy +s3tests_boto3/functional/test_iam.py::test_allow_object_actions_in_user_policy +s3tests_boto3/functional/test_iam.py::test_deny_object_actions_in_user_policy +s3tests_boto3/functional/test_iam.py::test_allow_multipart_actions_in_user_policy +s3tests_boto3/functional/test_iam.py::test_deny_multipart_actions_in_user_policy +s3tests_boto3/functional/test_iam.py::test_allow_tagging_actions_in_user_policy +s3tests_boto3/functional/test_iam.py::test_deny_tagging_actions_in_user_policy +s3tests_boto3/functional/test_iam.py::test_verify_conflicting_user_policy_statements +s3tests_boto3/functional/test_iam.py::test_verify_conflicting_user_policies +s3tests_boto3/functional/test_iam.py::test_verify_allow_iam_actions +s3tests_boto3/functional/test_sts.py::test_get_session_token +s3tests_boto3/functional/test_sts.py::test_get_session_token_permanent_creds_denied +s3tests_boto3/functional/test_sts.py::test_assume_role_allow +s3tests_boto3/functional/test_sts.py::test_assume_role_deny +s3tests_boto3/functional/test_sts.py::test_assume_role_creds_expiry +s3tests_boto3/functional/test_sts.py::test_assume_role_deny_head_nonexistent +s3tests_boto3/functional/test_sts.py::test_assume_role_allow_head_nonexistent +s3tests_boto3/functional/test_sts.py::test_assume_role_with_web_identity +s3tests_boto3/functional/test_sts.py::test_session_policy_check_on_different_buckets +s3tests_boto3/functional/test_sts.py::test_session_policy_check_on_same_bucket +s3tests_boto3/functional/test_sts.py::test_session_policy_check_put_obj_denial +s3tests_boto3/functional/test_sts.py::test_swapping_role_policy_and_session_policy +s3tests_boto3/functional/test_sts.py::test_session_policy_check_different_op_permissions +s3tests_boto3/functional/test_sts.py::test_session_policy_check_with_deny_effect +s3tests_boto3/functional/test_sts.py::test_session_policy_check_with_deny_on_same_op +s3tests_boto3/functional/test_sts.py::test_session_policy_bucket_policy_role_arn +s3tests_boto3/functional/test_sts.py::test_session_policy_bucket_policy_session_arn +s3tests_boto3/functional/test_sts.py::test_session_policy_copy_object +s3tests_boto3/functional/test_sts.py::test_session_policy_no_bucket_role_policy +s3tests_boto3/functional/test_sts.py::test_session_policy_bucket_policy_deny +s3tests_boto3/functional/test_sts.py::test_assume_role_with_web_identity_with_sub +s3tests_boto3/functional/test_sts.py::test_assume_role_with_web_identity_with_azp +s3tests_boto3/functional/test_sts.py::test_assume_role_with_web_identity_with_request_tag +s3tests_boto3/functional/test_sts.py::test_assume_role_with_web_identity_with_principal_tag +s3tests_boto3/functional/test_sts.py::test_assume_role_with_web_identity_for_all_values +s3tests_boto3/functional/test_sts.py::test_assume_role_with_web_identity_for_all_values_deny +s3tests_boto3/functional/test_sts.py::test_assume_role_with_web_identity_tag_keys_trust_policy +s3tests_boto3/functional/test_sts.py::test_assume_role_with_web_identity_tag_keys_role_policy +s3tests_boto3/functional/test_sts.py::test_assume_role_with_web_identity_resource_tag +s3tests_boto3/functional/test_sts.py::test_assume_role_with_web_identity_resource_tag_deny +s3tests_boto3/functional/test_sts.py::test_assume_role_with_web_identity_wrong_resource_tag_deny +s3tests_boto3/functional/test_sts.py::test_assume_role_with_web_identity_resource_tag_princ_tag +s3tests_boto3/functional/test_sts.py::test_assume_role_with_web_identity_resource_tag_copy_obj +s3tests_boto3/functional/test_sts.py::test_assume_role_with_web_identity_role_resource_tag +s3tests_boto3/functional/test_s3.py::test_cors_presigned_put_object_with_acl +s3tests_boto3/functional/test_s3.py::test_cors_presigned_put_object_tenant_with_acl +s3tests_boto3/functional/test_s3.py::test_put_bucket_acl_grant_group_read +s3tests_boto3/functional/test_s3.py::test_multipart_sse_c_get_part +s3tests_boto3/functional/test_s3.py::test_multipart_single_get_part +s3tests_boto3/functional/test_s3.py::test_non_multipart_sse_c_get_part +s3tests_boto3/functional/test_s3.py::test_cors_presigned_get_object_v2 +s3tests_boto3/functional/test_s3.py::test_cors_presigned_get_object_tenant_v2 +s3tests_boto3/functional/test_s3.py::test_cors_presigned_put_object_v2 +s3tests_boto3/functional/test_s3.py::test_cors_presigned_put_object_tenant_v2 +s3tests_boto3/functional/test_s3.py::test_lifecycle_delete +s3tests_boto3/functional/test_s3.py::test_lifecycle_deletemarker_expiration_with_days_tag +s3tests_boto3/functional/test_s3.py::test_encryption_sse_c_enforced_with_bucket_policy +s3tests_boto3/functional/test_s3.py::test_encryption_sse_c_deny_algo_with_bucket_policy +s3tests_boto3/functional/test_s3.py::test_bucket_policy_multipart +s3tests_boto3/functional/test_s3.py::test_bucket_policy_tenanted_bucket +s3tests_boto3/functional/test_s3.py::test_set_get_del_bucket_policy +s3tests_boto3/functional/test_s3.py::test_bucket_policy_upload_part_copy +s3tests_boto3/functional/test_s3.py::test_bucket_policy_put_obj_s3_incorrect_algo_sse_s3 +s3tests_boto3/functional/test_s3.py::test_object_lock_put_obj_lock_enable_after_create +s3tests_boto3/functional/test_s3.py::test_get_nonpublicpolicy_principal_bucket_policy_status +s3tests_boto3/functional/test_s3.py::test_block_public_policy_with_principal +s3tests_boto3/functional/test_s3.py::test_block_public_restrict_public_buckets +s3tests_boto3/functional/test_s3.py::test_put_get_delete_public_block +s3tests_boto3/functional/test_s3.py::test_object_checksum_sha256 +s3tests_boto3/functional/test_s3.py::test_object_checksum_crc64nvme +s3tests_boto3/functional/test_s3.py::test_multipart_checksum_sha256 +s3tests_boto3/functional/test_s3.py::test_multipart_use_cksum_helper_sha256 +s3tests_boto3/functional/test_s3.py::test_multipart_use_cksum_helper_crc64nvme +s3tests_boto3/functional/test_s3.py::test_multipart_use_cksum_helper_crc32 +s3tests_boto3/functional/test_s3.py::test_multipart_use_cksum_helper_crc32c +s3tests_boto3/functional/test_s3.py::test_multipart_use_cksum_helper_sha1 +s3tests_boto3/functional/test_s3.py::test_post_object_upload_checksum +s3tests_boto3/functional/test_s3.py::test_put_bucket_logging +s3tests_boto3/functional/test_s3.py::test_bucket_logging_mtime +s3tests_boto3/functional/test_s3.py::test_bucket_logging_simple_key +s3tests_boto3/functional/test_s3.py::test_bucket_logging_partitioned_key +s3tests_boto3/functional/test_s3.py::test_bucket_logging_bucket_auth_type +s3tests_boto3/functional/test_s3.py::test_bucket_logging_bucket_acl_required +s3tests_boto3/functional/test_s3.py::test_bucket_logging_object_acl_required +s3tests_boto3/functional/test_s3.py::test_put_bucket_logging_errors +s3tests_boto3/functional/test_s3.py::test_bucket_logging_owner +s3tests_boto3/functional/test_s3.py::test_put_bucket_logging_permissions +s3tests_boto3/functional/test_s3.py::test_put_bucket_logging_policy_wildcard +s3tests_boto3/functional/test_s3.py::test_put_bucket_logging_policy_wildcard_objects +s3tests_boto3/functional/test_s3.py::test_bucket_logging_permission_change_s +s3tests_boto3/functional/test_s3.py::test_put_bucket_logging_tenant_s +s3tests_boto3/functional/test_s3.py::test_put_bucket_logging_account_s +s3tests_boto3/functional/test_s3.py::test_rm_bucket_logging +s3tests_boto3/functional/test_s3.py::test_bucket_logging_put_objects +s3tests_boto3/functional/test_s3.py::test_bucket_logging_put_objects_versioned +s3tests_boto3/functional/test_s3.py::test_bucket_logging_put_concurrency +s3tests_boto3/functional/test_s3.py::test_bucket_logging_delete_objects +s3tests_boto3/functional/test_s3.py::test_bucket_logging_delete_objects_versioned +s3tests_boto3/functional/test_s3.py::test_bucket_logging_get_objects +s3tests_boto3/functional/test_s3.py::test_bucket_logging_get_objects_versioned +s3tests_boto3/functional/test_s3.py::test_bucket_logging_copy_objects +s3tests_boto3/functional/test_s3.py::test_bucket_logging_copy_objects_versioned +s3tests_boto3/functional/test_s3.py::test_bucket_logging_copy_objects_bucket +s3tests_boto3/functional/test_s3.py::test_bucket_logging_copy_objects_bucket_versioned +s3tests_boto3/functional/test_s3.py::test_bucket_logging_head_objects +s3tests_boto3/functional/test_s3.py::test_bucket_logging_head_objects_versioned +s3tests_boto3/functional/test_s3.py::test_bucket_logging_mpu_s +s3tests_boto3/functional/test_s3.py::test_bucket_logging_mpu_versioned_s +s3tests_boto3/functional/test_s3.py::test_bucket_logging_mpu_copy +s3tests_boto3/functional/test_s3.py::test_bucket_logging_mpu_copy_versioned +s3tests_boto3/functional/test_s3.py::test_bucket_logging_multi_delete +s3tests_boto3/functional/test_s3.py::test_bucket_logging_multi_delete_versioned +s3tests_boto3/functional/test_s3.py::test_bucket_logging_multiple_prefixes +s3tests_boto3/functional/test_s3.py::test_bucket_logging_single_prefix +s3tests_boto3/functional/test_s3.py::test_get_multipart_checksum_object_attributes +s3tests_boto3/functional/test_s3.py::test_get_multipart_object_attributes +s3tests_boto3/functional/test_s3.py::test_get_paginated_multipart_object_attributes +s3tests_boto3/functional/test_s3.py::test_get_single_multipart_object_attributes +s3tests_boto3/functional/test_s3.py::test_get_checksum_object_attributes +s3tests_boto3/functional/test_s3.py::test_get_versioned_object_attributes +s3tests_boto3/functional/test_s3.py::test_get_sse_c_encrypted_object_attributes +s3tests_boto3/functional/test_s3.py::test_get_object_attributes +s3tests_boto3/functional/test_s3.py::test_upload_part_copy_percent_encoded_key +s3tests_boto3/functional/test_s3.py::test_delete_marker_nonversioned +s3tests_boto3/functional/test_s3.py::test_delete_marker_versioned +s3tests_boto3/functional/test_s3.py::test_delete_marker_suspended +s3tests_boto3/functional/test_s3.py::test_delete_marker_expiration +s3tests_boto3/functional/test_sts.py::test_get_caller_identity_root +s3tests_boto3/functional/test_sts.py::test_get_caller_identity_non_root_user +s3tests_boto3/functional/test_sts.py::test_get_caller_identity_after_assume_role \ No newline at end of file diff --git a/src/test/external_tests/ceph_s3_tests/s3-tests-lists/ns_aws_s3_tests_pending_list.txt b/src/test/external_tests/ceph_s3_tests/s3-tests-lists/ns_aws_s3_tests_pending_list.txt new file mode 100644 index 0000000000..0d473badff --- /dev/null +++ b/src/test/external_tests/ceph_s3_tests/s3-tests-lists/ns_aws_s3_tests_pending_list.txt @@ -0,0 +1,117 @@ +s3tests_boto3/functional/test_s3.py::test_get_bucket_policy_status +s3tests/functional/test_s3.py::test_atomic_dual_conditional_write_1mb +s3tests_boto3/functional/test_s3.py::test_bucket_listv2_encoding_basic +s3tests_boto3/functional/test_s3.py::test_bucket_list_encoding_basic +s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_basic +s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_basic +s3tests_boto3/functional/test_s3.py::test_bucket_list_return_data +s3tests_boto3/functional/test_s3.py::test_account_usage +s3tests_boto3/functional/test_s3.py::test_head_bucket_usage +s3tests_boto3/functional/test_s3.py::test_post_object_invalid_signature +s3tests_boto3/functional/test_s3.py::test_post_object_invalid_access_key +s3tests_boto3/functional/test_s3.py::test_post_object_missing_policy_condition +s3tests_boto3/functional/test_s3.py::test_post_object_request_missing_policy_specified_field +s3tests_boto3/functional/test_s3.py::test_post_object_expired_policy +s3tests_boto3/functional/test_s3.py::test_post_object_invalid_request_field_value +s3tests_boto3/functional/test_s3.py::test_object_lock_put_obj_lock_invalid_mode +s3tests_boto3/functional/test_s3.py::test_object_lock_get_obj_retention_iso8601 +s3tests_boto3/functional/test_s3.py::test_object_lock_delete_object_with_retention_and_marker +s3tests_boto3/functional/test_s3.py::test_multipart_upload_on_a_bucket_with_policy +s3tests_boto3/functional/test_s3.py::test_multipart_upload_small +s3tests_boto3/functional/test_s3.py::test_post_object_authenticated_request_bad_access_key +s3tests_boto3/functional/test_s3.py::test_bucket_create_exists +s3tests_boto3/functional/test_s3.py::test_bucket_create_exists_nonowner +s3tests_boto3/functional/test_s3.py::test_bucket_recreate_overwrite_acl +s3tests_boto3/functional/test_s3.py::test_bucket_recreate_new_acl +s3tests_boto3/functional/test_s3.py::test_list_multipart_upload_owner +s3tests_boto3/functional/test_s3.py::test_lifecycle_expiration +s3tests_boto3/functional/test_s3.py::test_lifecycle_expiration_tags2 +s3tests_boto3/functional/test_s3.py::test_lifecycle_expiration_versioned_tags2 +s3tests_boto3/functional/test_s3.py::test_lifecycle_expiration_noncur_tags1 +s3tests_boto3/functional/test_s3.py::test_lifecycle_expiration_header_put +s3tests_boto3/functional/test_s3.py::test_lifecycle_expiration_header_head +s3tests_boto3/functional/test_s3.py::test_lifecycle_expiration_header_tags_head +s3tests_boto3/functional/test_s3.py::test_lifecycle_transition_set_invalid_date +s3tests_boto3/functional/test_s3.py::test_put_obj_enc_conflict_c_s3 +s3tests_boto3/functional/test_s3.py::test_put_obj_enc_conflict_c_kms +s3tests_boto3/functional/test_s3.py::test_put_obj_enc_conflict_s3_kms +s3tests_boto3/functional/test_s3.py::test_object_lock_multi_delete_object_with_retention +s3tests_boto3/functional/test_s3.py::test_object_lock_put_legal_hold +s3tests_boto3/functional/test_s3.py::test_object_lock_get_legal_hold +s3tests_boto3/functional/test_s3.py::test_object_lock_changing_mode_from_governance_with_bypass +s3tests_boto3/functional/test_s3.py::test_object_lock_changing_mode_from_governance_without_bypass +s3tests_boto3/functional/test_s3.py::test_object_lock_changing_mode_from_compliance +s3tests_boto3/functional/test_s3.py::test_object_read_unreadable +s3tests_boto3/functional/test_s3.py::test_get_public_acl_bucket_policy_status +s3tests_boto3/functional/test_s3.py::test_get_authpublic_acl_bucket_policy_status +s3tests_boto3/functional/test_s3.py::test_get_publicpolicy_acl_bucket_policy_status +s3tests_boto3/functional/test_s3.py::test_get_nonpublicpolicy_acl_bucket_policy_status +s3tests_boto3/functional/test_s3.py::test_put_public_block +s3tests_boto3/functional/test_s3.py::test_block_public_put_bucket_acls +s3tests_boto3/functional/test_s3.py::test_block_public_object_canned_acls +s3tests_boto3/functional/test_s3.py::test_block_public_policy +s3tests_boto3/functional/test_s3.py::test_ignore_public_acls +s3tests_boto3/functional/test_s3.py::test_sse_s3_default_post_object_authenticated_request +s3tests_boto3/functional/test_s3.py::test_sse_kms_default_post_object_authenticated_request +s3tests_boto3/functional/test_s3.py::test_object_write_with_chunked_transfer_encoding +s3tests_boto3/functional/test_s3.py::test_versioning_concurrent_multi_object_delete +s3tests_boto3/functional/test_s3.py::test_post_object_upload_size_rgw_chunk_size_bug +s3tests_boto3/functional/test_s3.py::test_get_object_torrent +s3tests/functional/test_headers.py::test_object_create_bad_md5_invalid_garbage_aws4 +s3tests/functional/test_headers.py::test_object_create_bad_contentlength_mismatch_below_aws4 +s3tests/functional/test_headers.py::test_object_create_bad_authorization_incorrect_aws4 +s3tests/functional/test_headers.py::test_object_create_bad_authorization_invalid_aws4 +s3tests/functional/test_headers.py::test_object_create_bad_ua_empty_aws4 +s3tests/functional/test_headers.py::test_object_create_bad_ua_none_aws4 +s3tests/functional/test_headers.py::test_object_create_bad_date_invalid_aws4 +s3tests/functional/test_headers.py::test_object_create_bad_amz_date_invalid_aws4 +s3tests/functional/test_headers.py::test_object_create_bad_date_empty_aws4 +s3tests/functional/test_headers.py::test_object_create_bad_amz_date_empty_aws4 +s3tests/functional/test_headers.py::test_object_create_bad_date_none_aws4 +s3tests/functional/test_headers.py::test_object_create_bad_amz_date_none_aws4 +s3tests/functional/test_headers.py::test_object_create_bad_date_before_today_aws4 +s3tests/functional/test_headers.py::test_object_create_bad_amz_date_before_today_aws4 +s3tests/functional/test_headers.py::test_object_create_bad_date_after_today_aws4 +s3tests/functional/test_headers.py::test_object_create_bad_amz_date_after_today_aws4 +s3tests/functional/test_headers.py::test_object_create_bad_date_before_epoch_aws4 +s3tests/functional/test_headers.py::test_object_create_bad_amz_date_before_epoch_aws4 +s3tests/functional/test_headers.py::test_object_create_bad_date_after_end_aws4 +s3tests/functional/test_headers.py::test_object_create_bad_amz_date_after_end_aws4 +s3tests/functional/test_headers.py::test_object_create_missing_signed_custom_header_aws4 +s3tests/functional/test_headers.py::test_object_create_missing_signed_header_aws4 +s3tests/functional/test_headers.py::test_bucket_create_bad_authorization_invalid_aws4 +s3tests/functional/test_headers.py::test_bucket_create_bad_ua_empty_aws4 +s3tests/functional/test_headers.py::test_bucket_create_bad_ua_none_aws4 +s3tests/functional/test_headers.py::test_bucket_create_bad_date_invalid_aws4 +s3tests/functional/test_headers.py::test_bucket_create_bad_amz_date_invalid_aws4 +s3tests/functional/test_headers.py::test_bucket_create_bad_date_empty_aws4 +s3tests/functional/test_headers.py::test_bucket_create_bad_amz_date_empty_aws4 +s3tests/functional/test_headers.py::test_bucket_create_bad_date_none_aws4 +s3tests/functional/test_headers.py::test_bucket_create_bad_amz_date_none_aws4 +s3tests/functional/test_headers.py::test_bucket_create_bad_date_before_today_aws4 +s3tests/functional/test_headers.py::test_bucket_create_bad_amz_date_before_today_aws4 +s3tests/functional/test_headers.py::test_bucket_create_bad_date_after_today_aws4 +s3tests/functional/test_headers.py::test_bucket_create_bad_amz_date_after_today_aws4 +s3tests/functional/test_headers.py::test_bucket_create_bad_date_before_epoch_aws4 +s3tests/functional/test_headers.py::test_bucket_create_bad_amz_date_before_epoch_aws4 +s3tests_boto3/functional/test_s3.py::test_post_object_wrong_bucket +s3tests_boto3/functional/test_s3.py::test_object_raw_get_x_amz_expires_not_expired_tenant +s3tests_boto3/functional/test_s3.py::test_cors_presigned_get_object_tenant +s3tests_boto3/functional/test_s3.py::test_multipart_get_part +s3tests_boto3/functional/test_s3.py::test_non_multipart_get_part +s3tests_boto3/functional/test_s3.py::test_cors_presigned_put_object_tenant +s3tests_boto3/functional/test_s3.py::test_object_presigned_put_object_with_acl_tenant +s3tests_boto3/functional/test_s3.py::test_lifecycle_expiration_newer_noncurrent +s3tests_boto3/functional/test_s3.py::test_lifecycle_expiration_size_gt +s3tests_boto3/functional/test_s3.py::test_lifecycle_expiration_size_lt +s3tests_boto3/functional/test_s3.py::test_object_lock_delete_multipart_object_with_retention +s3tests_boto3/functional/test_s3.py::test_object_lock_delete_multipart_object_with_legal_hold_on +s3tests_boto3/functional/test_s3.py::test_get_undefined_public_block +s3tests_boto3/functional/test_s3.py::test_get_public_block_deny_bucket_policy +s3tests_boto3/functional/test_s3.py::test_get_bucket_encryption_s3 +s3tests_boto3/functional/test_s3.py::test_get_bucket_encryption_kms +s3tests_boto3/functional/test_s3.py::test_delete_bucket_encryption_s3 +s3tests_boto3/functional/test_s3.py::test_delete_bucket_encryption_kms +s3tests_boto3/functional/test_s3.py::test_lifecycle_expiration_tags1 +s3tests_boto3/functional/test_headers.py::test_bucket_create_bad_ua_empty_aws2 +s3tests_boto3/functional/test_headers.py::test_bucket_create_bad_ua_none_aws2 \ No newline at end of file diff --git a/src/test/external_tests/ceph_s3_tests/test_ceph_s3_config_and_run_s3_tests.sh b/src/test/external_tests/ceph_s3_tests/test_ceph_s3_config_and_run_s3_tests.sh index 10f3cd6aaf..d0778c20e3 100755 --- a/src/test/external_tests/ceph_s3_tests/test_ceph_s3_config_and_run_s3_tests.sh +++ b/src/test/external_tests/ceph_s3_tests/test_ceph_s3_config_and_run_s3_tests.sh @@ -8,8 +8,17 @@ set -x CEPH_S3_DIRECTORY="src/test/external_tests/ceph_s3_tests" CEPH_S3_TESTS_CONFIG="${CEPH_S3_DIRECTORY}/test_ceph_s3_config_setup.js" CEPH_S3_RUN_TESTS="${CEPH_S3_DIRECTORY}/test_ceph_s3.js" -S3_CEPH_TEST_BLACKLIST="${CEPH_S3_DIRECTORY}/s3-tests-lists/s3_tests_black_list.txt" -S3_CEPH_TEST_PENDING_LIST="${CEPH_S3_DIRECTORY}/s3-tests-lists/s3_tests_pending_list.txt" + +# choose test lists based on USE_S3_NAMESPACE_RESOURCE flag +# later we will add more test lists for other types of namespace resources +if [ "${USE_S3_NAMESPACE_RESOURCE}" = "true" ]; then + S3_CEPH_TEST_BLACKLIST="${CEPH_S3_DIRECTORY}/s3-tests-lists/ns_aws_s3_tests_black_list.txt" + S3_CEPH_TEST_PENDING_LIST="${CEPH_S3_DIRECTORY}/s3-tests-lists/ns_aws_s3_tests_pending_list.txt" +else + S3_CEPH_TEST_BLACKLIST="${CEPH_S3_DIRECTORY}/s3-tests-lists/s3_tests_black_list.txt" + S3_CEPH_TEST_PENDING_LIST="${CEPH_S3_DIRECTORY}/s3-tests-lists/s3_tests_pending_list.txt" +fi + NUMBER_OF_WORKERS=1 cd /root/node_modules/noobaa-core/ diff --git a/src/test/external_tests/ceph_s3_tests/test_ceph_s3_config_setup.js b/src/test/external_tests/ceph_s3_tests/test_ceph_s3_config_setup.js index 5a58939380..f41b69c378 100644 --- a/src/test/external_tests/ceph_s3_tests/test_ceph_s3_config_setup.js +++ b/src/test/external_tests/ceph_s3_tests/test_ceph_s3_config_setup.js @@ -14,6 +14,8 @@ dbg.set_process_name('test_ceph_s3'); const os_utils = require('../../../util/os_utils'); const api = require('../../../api'); const { CEPH_TEST } = require('./test_ceph_s3_constants.js'); +const AWS = require('aws-sdk'); +const cloud_utils = require('../../../util/cloud_utils'); // create a global RPC client // the client is used to perform setup operations on noobaa system @@ -60,18 +62,38 @@ async function ceph_test_setup() { console.log('conf file updated'); let system = await client.system.read_system(); - // We are taking the first host pool, in normal k8s setup is default backing store - const test_pool = system.pools.filter(p => p.resource_type === 'HOSTS')[0]; - console.log(test_pool); + try { + const use_s3_namespace = process.env.USE_S3_NAMESPACE_RESOURCE === 'true'; + + const default_resource = use_s3_namespace ? + await setup_s3_namespace_resource() : + (() => { + // We are taking the first host pool, in normal k8s setup is default backing store + const test_pool = system.pools.filter(p => p.resource_type === 'HOSTS')[0]; + console.log(test_pool); + return test_pool.name; + })(); + console.log("default_resource: ", default_resource); + + const account_config = use_s3_namespace ? { + cephalt: { nsfs_account_config: CEPH_TEST.ns_aws_cephalt_account_config }, + cephtenant: { nsfs_account_config: CEPH_TEST.ns_aws_cephtenant_account_config } + } : { + cephalt: {}, + cephtenant: {} + }; + await client.account.create_account({ ...CEPH_TEST.new_account_params, - default_resource: test_pool.name + default_resource: default_resource, + ...account_config.cephalt }); await client.account.create_account({ ...CEPH_TEST.new_account_params_tenant, - default_resource: test_pool.name + default_resource: default_resource, + ...account_config.cephtenant }); } catch (err) { console.log("Failed to create account or tenant, assuming they were already created and continuing. ", err.message); @@ -103,6 +125,59 @@ async function ceph_test_setup() { } } +async function setup_s3_namespace_resource() { + const minio_endpoint = process.env.MINIO_ENDPOINT; + const minio_user = process.env.MINIO_USER; + const minio_password = process.env.MINIO_PASSWORD; + const minio_test_bucket = process.env.MINIO_TEST_BUCKET; + const namespace_resource_name = "ns-aws"; + + // create the bucket in Minio using AWS SDK + console.info(`Creating bucket ${minio_test_bucket} in Minio...`); + try { + const s3 = new AWS.S3({ + endpoint: minio_endpoint, + accessKeyId: minio_user, + secretAccessKey: minio_password, + s3ForcePathStyle: true, + signatureVersion: cloud_utils.get_s3_endpoint_signature_ver(minio_endpoint) + }); + + await s3.createBucket({ Bucket: minio_test_bucket }).promise(); + console.log(`Bucket ${minio_test_bucket} created successfully in Minio`); + } catch (err) { + throw new Error(`Bucket creation failed: ${err.message}`); + } + + console.info('Creating external connection...'); + try { + await client.account.add_external_connection({ + name: "minio-connection", + endpoint: minio_endpoint, + endpoint_type: "S3_COMPATIBLE", + identity: minio_user, + secret: minio_password + }); + console.log('External connection created successfully'); + } catch (err) { + throw new Error(`External connection creation failed: ${err.message}`); + } + + console.info('Creating namespace resource...'); + try { + await client.pool.create_namespace_resource({ + name: namespace_resource_name, + connection: "minio-connection", + target_bucket: minio_test_bucket + }); + console.log('Namespace resource created successfully'); + } catch (err) { + throw new Error(`Namespace resource creation failed: ${err.message}`); + } + + return namespace_resource_name; +} + if (require.main === module) { main(); } diff --git a/src/test/external_tests/ceph_s3_tests/test_ceph_s3_constants.js b/src/test/external_tests/ceph_s3_tests/test_ceph_s3_constants.js index 6960b9b3a6..dd8f8c687b 100644 --- a/src/test/external_tests/ceph_s3_tests/test_ceph_s3_constants.js +++ b/src/test/external_tests/ceph_s3_tests/test_ceph_s3_constants.js @@ -41,6 +41,18 @@ const CEPH_TEST = { anonymous: true, uid: process.getuid(), gid: process.getgid() + }, + ns_aws_cephalt_account_config: { + uid: 1000, + gid: 1000, + new_buckets_path: '/tmp/ns_aws_ceph_alt', + nsfs_only: false + }, + ns_aws_cephtenant_account_config: { + uid: 2000, + gid: 2000, + new_buckets_path: '/tmp/ns_aws_ceph_tenant', + nsfs_only: false } }; const DEFAULT_NUMBER_OF_WORKERS = 5; //5 was the number of workers in the previous CI/CD process