From 5df871847b6a4dd6d43722725b007bfd217ba8f1 Mon Sep 17 00:00:00 2001 From: Rachel Hagerman <110480692+rlhagerm@users.noreply.github.com> Date: Fri, 7 Mar 2025 13:16:33 -0600 Subject: [PATCH 01/10] Adding a test code examples section. --- botocore/docs/bcdoc/restdoc.py | 3 +- botocore/docs/codeexamples.py | 71 +++++++++++++++++ botocore/docs/service.py | 26 +++++++ botocore/loaders.py | 137 +++++++++++++++++++++++++++++++++ 4 files changed, 236 insertions(+), 1 deletion(-) create mode 100644 botocore/docs/codeexamples.py diff --git a/botocore/docs/bcdoc/restdoc.py b/botocore/docs/bcdoc/restdoc.py index 3868126cc9..5311b8c0aa 100644 --- a/botocore/docs/bcdoc/restdoc.py +++ b/botocore/docs/bcdoc/restdoc.py @@ -44,6 +44,7 @@ 'client-api': 4, 'paginator-api': 3, 'waiter-api': 3, + 'examples-api': 3, } @@ -203,7 +204,7 @@ def add_new_section(self, name, context=None): name=name, target=self.target, context=context ) section.path = self.path + [name] - # Indent the section apporpriately as well + # Indent the section appropriately as well section.style.indentation = self.style.indentation section.translation_map = self.translation_map section.hrefs = self.hrefs diff --git a/botocore/docs/codeexamples.py b/botocore/docs/codeexamples.py new file mode 100644 index 0000000000..ac3a22d394 --- /dev/null +++ b/botocore/docs/codeexamples.py @@ -0,0 +1,71 @@ +# Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +import os + +from botocore import xform_name +from botocore.compat import OrderedDict +from botocore.docs.bcdoc.restdoc import DocumentStructure +from botocore.docs.method import document_model_driven_method +from botocore.docs.utils import DocumentedShape +from botocore.utils import get_service_module_name + + +class CodeExamplesDocumenter: + def __init__(self, client, root_docs_path): + self._client = client + self._client_class_name = self._client.__class__.__name__ + self._service_name = self._client.meta.service_model.service_name + self._root_docs_path = root_docs_path + self._USER_GUIDE_LINK = ( + 'https://boto3.amazonaws.com/' + 'v1/documentation/api/latest/guide/clients.html#waiters' + ) + self._CODE_EXAMPLE_LINK = ( + 'https://docs.aws.amazon.com/code-library/latest/ug/python_3_' + ) + + def document_code_examples(self, section, examples): + """Documents the code library code examples for a service. + + :param section: The section to write to. + :param examples: The list of examples. + """ + section.style.h2('AWS Code Examples') + self._add_overview(section) + + # List the available Code Library examples with a link. + # TODO: fix service name + for example in examples: + section.style.new_paragraph() + title_text = examples[example]['title'] + plain_title = title_text.replace('', '').replace('', '') + section.style.external_link( + title=plain_title, + link=examples[example]['doc_filenames']['service_pages']['lookoutvision'], + # link=self._CODE_EXAMPLE_LINK + self._service_name + '_code_examples.html', + ) + section.style.new_line() + + def _add_overview(self, section): + section.style.new_line() + section.write( + 'Explore more examples for this service in the ' + ) + section.style.external_link( + title='AWS Examples Code Library', + link=self._CODE_EXAMPLE_LINK + self._service_name + '_code_examples.html', + ) + section.write('.') + section.style.new_line() + + diff --git a/botocore/docs/service.py b/botocore/docs/service.py index d20a889dc9..77b98e1e54 100644 --- a/botocore/docs/service.py +++ b/botocore/docs/service.py @@ -18,6 +18,7 @@ ) from botocore.docs.paginator import PaginatorDocumenter from botocore.docs.waiter import WaiterDocumenter +from botocore.docs.codeexamples import CodeExamplesDocumenter from botocore.exceptions import DataNotFoundError @@ -42,6 +43,7 @@ def __init__(self, service_name, session, root_docs_path): 'paginator-api', 'waiter-api', 'client-context-params', + 'examples-api', ] def document_service(self): @@ -61,6 +63,7 @@ def document_service(self): 'client-context-params' ) self.client_context_params(context_params_section) + self.examples_api(doc_structure.get_section('examples-api')) return doc_structure.flush_structure() def title(self, section): @@ -111,6 +114,29 @@ def waiter_api(self, section): ) waiter_documenter.document_waiters(section) + def examples_api(self, section): + library_examples_list = None + try: + library_examples_list = self.get_library_examples() + except DataNotFoundError: + pass + + if library_examples_list: + examples_documenter = CodeExamplesDocumenter( + self._client, self._root_docs_path + ) + examples_documenter.document_code_examples(section, library_examples_list) + + def get_library_examples(self): + loader = self._session.get_component('data_loader') + examples = loader.file_loader.load_examples_file('sample_file') + print(examples['examples']) + + for example in examples['examples']: + print(example) + print(examples['examples'][example]['id']) + return examples['examples'] + def get_examples(self, service_name, api_version=None): loader = self._session.get_component('data_loader') examples = loader.load_service_model( diff --git a/botocore/loaders.py b/botocore/loaders.py index f5072a3e5f..2b0578c9f6 100644 --- a/botocore/loaders.py +++ b/botocore/loaders.py @@ -197,6 +197,143 @@ def load_file(self, file_path): return data return None + def load_examples_file(self, file_path): + """Load the examples data if available + + :return: The loaded data if it exists, otherwise None. + + """ + examples_json = '''{ + "examples": { + "lookoutvision_CreateDataset": { + "id": "lookoutvision_CreateDataset", + "file": "lookoutvision_metadata.yaml", + "languages": { + "Python": { + "name": "Python", + "property": "", + "versions": [ + { + "sdk_version": 3, + "block_content": null, + "excerpts": [ + { + "description": null, + "snippet_tags": [ + "python.example_code.lookoutvision.Datasets", + "python.example_code.lookoutvision.CreateDataset" + ], + "snippet_files": [], + "genai": "none" + } + ], + "github": "python/example_code/lookoutvision", + "sdkguide": null, + "more_info": [] + } + ] + } + }, + "title": "Use CreateDataset", + "title_abbrev": "CreateDataset", + "synopsis": "use CreateDataset.", + "category": "Api", + "guide_topic": { + "title": "Creating your dataset", + "url": "lookout-for-vision/latest/developer-guide/model-create-dataset.html" + }, + "service_main": null, + "services": { + "lookoutvision": { + "__set__": [ + "CreateDataset" + ] + } + }, + "doc_filenames": { + "service_pages": { + "lookoutvision": "https://docs.aws.amazon.com/code-library/latest/ug/lookoutvision_example_lookoutvision_CreateDataset_section.html" + }, + "sdk_pages": { + "": { + "3": { + "actions_scenarios": { + "lookoutvision": "https://docs.aws.amazon.com/code-library/latest/ug/_3_lookoutvision_code_examples.html#actions" + }, + "cross_service": null + } + } + } + }, + "synopsis_list": [], + "source_key": null + }, + "lookoutvision_Scenario_CreateManifestFile": { + "id": "lookoutvision_Scenario_CreateManifestFile", + "file": "lookoutvision_metadata.yaml", + "languages": { + "Python": { + "name": "Python", + "property": "", + "versions": [ + { + "sdk_version": 3, + "block_content": null, + "excerpts": [ + { + "description": null, + "snippet_tags": [ + "python.example_code.lookoutvision.Datasets", + "python.example_code.lookoutvision.Scenario_CreateManifestFile" + ], + "snippet_files": [], + "genai": "none" + } + ], + "github": "python/example_code/lookoutvision", + "sdkguide": null, + "more_info": [] + } + ] + } + }, + "title": "Create a Lookout for Vision manifest file using an AWS SDK", + "title_abbrev": "Create a manifest file", + "synopsis": "create a Lookout for Vision manifest file and upload it to Amazon S3.", + "category": "Scenarios", + "guide_topic": { + "title": "Creating a manifest file", + "url": "lookout-for-vision/latest/developer-guide/manifest-files.html" + }, + "service_main": null, + "services": { + "lookoutvision": { + "__set__": [] + } + }, + "doc_filenames": { + "service_pages": { + "lookoutvision": "https://docs.aws.amazon.com/code-library/latest/ug/lookoutvision_example_lookoutvision_Scenario_CreateManifestFile_section.html" + }, + "sdk_pages": { + "": { + "3": { + "actions_scenarios": { + "lookoutvision": "https://docs.aws.amazon.com/code-library/latest/ug/_3_lookoutvision_code_examples.html#scenarios" + }, + "cross_service": null + } + } + } + }, + "synopsis_list": [], + "source_key": null + } + } + }''' + # return json.loads(examples_json, object_pairs_hook=OrderedDict) + return json.loads(examples_json) + def create_loader(search_path_string=None): """Create a Loader class. From 4dc960f220bf4b95f57a94538d8ee1e0edec7d00 Mon Sep 17 00:00:00 2001 From: Rachel Hagerman <110480692+rlhagerm@users.noreply.github.com> Date: Fri, 7 Mar 2025 13:32:48 -0600 Subject: [PATCH 02/10] Update code examples section --- botocore/docs/codeexamples.py | 8 ++++---- botocore/docs/service.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/botocore/docs/codeexamples.py b/botocore/docs/codeexamples.py index ac3a22d394..558eace5e3 100644 --- a/botocore/docs/codeexamples.py +++ b/botocore/docs/codeexamples.py @@ -40,13 +40,13 @@ def document_code_examples(self, section, examples): :param section: The section to write to. :param examples: The list of examples. """ - section.style.h2('AWS Code Examples') + section.style.h2('AWS SDK Code Examples') self._add_overview(section) # List the available Code Library examples with a link. # TODO: fix service name for example in examples: - section.style.new_paragraph() + section.style.start_li() title_text = examples[example]['title'] plain_title = title_text.replace('', '').replace('', '') section.style.external_link( @@ -54,7 +54,7 @@ def document_code_examples(self, section, examples): link=examples[example]['doc_filenames']['service_pages']['lookoutvision'], # link=self._CODE_EXAMPLE_LINK + self._service_name + '_code_examples.html', ) - section.style.new_line() + section.style.end_li() def _add_overview(self, section): section.style.new_line() @@ -62,7 +62,7 @@ def _add_overview(self, section): 'Explore more examples for this service in the ' ) section.style.external_link( - title='AWS Examples Code Library', + title='AWS SDK Code Examples Code Library', link=self._CODE_EXAMPLE_LINK + self._service_name + '_code_examples.html', ) section.write('.') diff --git a/botocore/docs/service.py b/botocore/docs/service.py index 77b98e1e54..ce0a9f6ffe 100644 --- a/botocore/docs/service.py +++ b/botocore/docs/service.py @@ -130,11 +130,11 @@ def examples_api(self, section): def get_library_examples(self): loader = self._session.get_component('data_loader') examples = loader.file_loader.load_examples_file('sample_file') - print(examples['examples']) + # print(examples['examples']) - for example in examples['examples']: - print(example) - print(examples['examples'][example]['id']) + # for example in examples['examples']: + # print(example) + # print(examples['examples'][example]['id']) return examples['examples'] def get_examples(self, service_name, api_version=None): From 51f61c9a920cbe032453a65d95ed9adc4b520c39 Mon Sep 17 00:00:00 2001 From: Rachel Hagerman <110480692+rlhagerm@users.noreply.github.com> Date: Mon, 24 Mar 2025 16:54:42 -0500 Subject: [PATCH 03/10] Add categories and service map. --- botocore/docs/codeexamples.py | 36 +++++--- botocore/docs/service.py | 25 ++++-- botocore/loaders.py | 151 +++++----------------------------- botocore/utils.py | 4 + 4 files changed, 64 insertions(+), 152 deletions(-) diff --git a/botocore/docs/codeexamples.py b/botocore/docs/codeexamples.py index 558eace5e3..7494a4044d 100644 --- a/botocore/docs/codeexamples.py +++ b/botocore/docs/codeexamples.py @@ -34,32 +34,42 @@ def __init__(self, client, root_docs_path): 'https://docs.aws.amazon.com/code-library/latest/ug/python_3_' ) - def document_code_examples(self, section, examples): + def document_code_examples(self, section, examples, service_id): """Documents the code library code examples for a service. :param section: The section to write to. :param examples: The list of examples. + :param service_id: The code examples service id. """ section.style.h2('AWS SDK Code Examples') self._add_overview(section) # List the available Code Library examples with a link. - # TODO: fix service name - for example in examples: - section.style.start_li() - title_text = examples[example]['title'] - plain_title = title_text.replace('', '').replace('', '') - section.style.external_link( - title=plain_title, - link=examples[example]['doc_filenames']['service_pages']['lookoutvision'], - # link=self._CODE_EXAMPLE_LINK + self._service_name + '_code_examples.html', - ) - section.style.end_li() + + # Group the examples by category. Do not show a category if there are no examples. + example_categories = {} + for i in range(len(examples)): + example_categories.setdefault(examples[i]['category'], []).append(examples[i]) + + # for example in examples: + for category in example_categories: + section.style.new_line() + section.style.h3(category) + for i in range(len(example_categories[category])): + section.style.start_li() + title_text = example_categories[category][i]['title'] + if not title_text: + title_text = example_categories[category][i]['id'].rsplit('_', 1)[-1] + section.style.external_link( + title=title_text, + link=example_categories[category][i]['doc_filenames']['service_pages'][service_id], + ) + section.style.end_li() def _add_overview(self, section): section.style.new_line() section.write( - 'Explore more examples for this service in the ' + 'Explore code examples in the ' ) section.style.external_link( title='AWS SDK Code Examples Code Library', diff --git a/botocore/docs/service.py b/botocore/docs/service.py index ce0a9f6ffe..61afe68581 100644 --- a/botocore/docs/service.py +++ b/botocore/docs/service.py @@ -20,6 +20,9 @@ from botocore.docs.waiter import WaiterDocumenter from botocore.docs.codeexamples import CodeExamplesDocumenter from botocore.exceptions import DataNotFoundError +from botocore.utils import ( + SDK_CODE_EXAMPLE_ALIASES +) class ServiceDocumenter: @@ -117,7 +120,8 @@ def waiter_api(self, section): def examples_api(self, section): library_examples_list = None try: - library_examples_list = self.get_library_examples() + code_example_service_name = SDK_CODE_EXAMPLE_ALIASES.get(self._service_name, self._service_name) + library_examples_list = self.get_library_examples(code_example_service_name) except DataNotFoundError: pass @@ -125,17 +129,20 @@ def examples_api(self, section): examples_documenter = CodeExamplesDocumenter( self._client, self._root_docs_path ) - examples_documenter.document_code_examples(section, library_examples_list) + examples_documenter.document_code_examples(section, library_examples_list, code_example_service_name) - def get_library_examples(self): + def get_library_examples(self, service_name): loader = self._session.get_component('data_loader') - examples = loader.file_loader.load_examples_file('sample_file') - # print(examples['examples']) + #TODO: move these constants to config somewhere + git_url = 'https://raw.githubusercontent.com/rlhagerm/aws-doc-sdk-examples/fe9f3a68c94ba888f96e23e799aa8b43cb3f457d/python/example_code/' + git_service_url = f"{git_url}/{service_name}/examples_catalog.json" - # for example in examples['examples']: - # print(example) - # print(examples['examples'][example]['id']) - return examples['examples'] + examples = loader.file_loader.load_examples_url(git_service_url) + + if len(examples) > 0: + return examples['examples'] + + return [] def get_examples(self, service_name, api_version=None): loader = self._session.get_component('data_loader') diff --git a/botocore/loaders.py b/botocore/loaders.py index 2b0578c9f6..36167f5391 100644 --- a/botocore/loaders.py +++ b/botocore/loaders.py @@ -104,6 +104,7 @@ import logging import os +import requests from botocore import BOTOCORE_ROOT from botocore.compat import HAS_GZIP, OrderedDict, json @@ -203,136 +204,26 @@ def load_examples_file(self, file_path): :return: The loaded data if it exists, otherwise None. """ - examples_json = '''{ - "examples": { - "lookoutvision_CreateDataset": { - "id": "lookoutvision_CreateDataset", - "file": "lookoutvision_metadata.yaml", - "languages": { - "Python": { - "name": "Python", - "property": "", - "versions": [ - { - "sdk_version": 3, - "block_content": null, - "excerpts": [ - { - "description": null, - "snippet_tags": [ - "python.example_code.lookoutvision.Datasets", - "python.example_code.lookoutvision.CreateDataset" - ], - "snippet_files": [], - "genai": "none" - } - ], - "github": "python/example_code/lookoutvision", - "sdkguide": null, - "more_info": [] - } - ] - } - }, - "title": "Use CreateDataset", - "title_abbrev": "CreateDataset", - "synopsis": "use CreateDataset.", - "category": "Api", - "guide_topic": { - "title": "Creating your dataset", - "url": "lookout-for-vision/latest/developer-guide/model-create-dataset.html" - }, - "service_main": null, - "services": { - "lookoutvision": { - "__set__": [ - "CreateDataset" - ] - } - }, - "doc_filenames": { - "service_pages": { - "lookoutvision": "https://docs.aws.amazon.com/code-library/latest/ug/lookoutvision_example_lookoutvision_CreateDataset_section.html" - }, - "sdk_pages": { - "": { - "3": { - "actions_scenarios": { - "lookoutvision": "https://docs.aws.amazon.com/code-library/latest/ug/_3_lookoutvision_code_examples.html#actions" - }, - "cross_service": null - } - } - } - }, - "synopsis_list": [], - "source_key": null - }, - "lookoutvision_Scenario_CreateManifestFile": { - "id": "lookoutvision_Scenario_CreateManifestFile", - "file": "lookoutvision_metadata.yaml", - "languages": { - "Python": { - "name": "Python", - "property": "", - "versions": [ - { - "sdk_version": 3, - "block_content": null, - "excerpts": [ - { - "description": null, - "snippet_tags": [ - "python.example_code.lookoutvision.Datasets", - "python.example_code.lookoutvision.Scenario_CreateManifestFile" - ], - "snippet_files": [], - "genai": "none" - } - ], - "github": "python/example_code/lookoutvision", - "sdkguide": null, - "more_info": [] - } - ] - } - }, - "title": "Create a Lookout for Vision manifest file using an AWS SDK", - "title_abbrev": "Create a manifest file", - "synopsis": "create a Lookout for Vision manifest file and upload it to Amazon S3.", - "category": "Scenarios", - "guide_topic": { - "title": "Creating a manifest file", - "url": "lookout-for-vision/latest/developer-guide/manifest-files.html" - }, - "service_main": null, - "services": { - "lookoutvision": { - "__set__": [] - } - }, - "doc_filenames": { - "service_pages": { - "lookoutvision": "https://docs.aws.amazon.com/code-library/latest/ug/lookoutvision_example_lookoutvision_Scenario_CreateManifestFile_section.html" - }, - "sdk_pages": { - "": { - "3": { - "actions_scenarios": { - "lookoutvision": "https://docs.aws.amazon.com/code-library/latest/ug/_3_lookoutvision_code_examples.html#scenarios" - }, - "cross_service": null - } - } - } - }, - "synopsis_list": [], - "source_key": null - } - } - }''' - # return json.loads(examples_json, object_pairs_hook=OrderedDict) - return json.loads(examples_json) + + examples_json = self.load_file(file_path) + if examples_json: + return examples_json + return [] + + def load_examples_url(self, file_url): + """Load the examples data from a url if available + + :return: The loaded data if it exists, otherwise None. + + """ + response = requests.get(file_url) + if response.status_code == 200: + example_json = response.text + if example_json: + examples = json.loads(example_json) + return examples + + return [] def create_loader(search_path_string=None): diff --git a/botocore/utils.py b/botocore/utils.py index 30a513ea90..e2fd92cea7 100644 --- a/botocore/utils.py +++ b/botocore/utils.py @@ -178,6 +178,10 @@ "tagging": "resource-groups-tagging-api", } +# The SDK code examples library service names mapped to the service id, if they do not match. +SDK_CODE_EXAMPLE_ALIASES = { + "apigateway": "api-gateway", +} # This pattern can be used to detect if a header is a flexible checksum header CHECKSUM_HEADER_PATTERN = re.compile( From 469bc653aed22050dda41a979a030e1b8aa58a99 Mon Sep 17 00:00:00 2001 From: Rachel Hagerman <110480692+rlhagerm@users.noreply.github.com> Date: Tue, 25 Mar 2025 10:11:34 -0500 Subject: [PATCH 04/10] Updates to service page logic. --- botocore/docs/codeexamples.py | 4 +++- botocore/loaders.py | 1 + botocore/utils.py | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/botocore/docs/codeexamples.py b/botocore/docs/codeexamples.py index 7494a4044d..41f1958d8c 100644 --- a/botocore/docs/codeexamples.py +++ b/botocore/docs/codeexamples.py @@ -60,9 +60,11 @@ def document_code_examples(self, section, examples, service_id): title_text = example_categories[category][i]['title'] if not title_text: title_text = example_categories[category][i]['id'].rsplit('_', 1)[-1] + service_page = example_categories[category][i]['doc_filenames']['service_pages'].get( + service_id,list(example_categories[category][i]['doc_filenames']['service_pages'].values())[0]) section.style.external_link( title=title_text, - link=example_categories[category][i]['doc_filenames']['service_pages'][service_id], + link=service_page, ) section.style.end_li() diff --git a/botocore/loaders.py b/botocore/loaders.py index 36167f5391..fedcbb4d97 100644 --- a/botocore/loaders.py +++ b/botocore/loaders.py @@ -345,6 +345,7 @@ def list_available_services(self, type_name): if self.file_loader.exists(full_load_path): services.add(service_name) break + print(sorted(services)) return sorted(services) @instance_cache diff --git a/botocore/utils.py b/botocore/utils.py index e2fd92cea7..2d69c24d11 100644 --- a/botocore/utils.py +++ b/botocore/utils.py @@ -181,6 +181,9 @@ # The SDK code examples library service names mapped to the service id, if they do not match. SDK_CODE_EXAMPLE_ALIASES = { "apigateway": "api-gateway", + "autoscaling": "auto-scaling", + "cognito-identity": "cognito", + "kinesis-analytics-v2": "kinesisanalyticsv2", } # This pattern can be used to detect if a header is a flexible checksum header From b970473520a516c4afe20e528a0b746e1b064877 Mon Sep 17 00:00:00 2001 From: Rachel Hagerman <110480692+rlhagerm@users.noreply.github.com> Date: Wed, 26 Mar 2025 10:40:37 -0500 Subject: [PATCH 05/10] Updates to naming and tests. --- botocore/docs/bcdoc/restdoc.py | 2 +- botocore/docs/codeexamples.py | 48 +++++++----- botocore/docs/service.py | 32 +++----- botocore/loaders.py | 1 - tests/unit/docs/test_sdkcodeexamples.py | 99 +++++++++++++++++++++++++ 5 files changed, 138 insertions(+), 44 deletions(-) create mode 100644 tests/unit/docs/test_sdkcodeexamples.py diff --git a/botocore/docs/bcdoc/restdoc.py b/botocore/docs/bcdoc/restdoc.py index 5311b8c0aa..c24f1d698a 100644 --- a/botocore/docs/bcdoc/restdoc.py +++ b/botocore/docs/bcdoc/restdoc.py @@ -44,7 +44,7 @@ 'client-api': 4, 'paginator-api': 3, 'waiter-api': 3, - 'examples-api': 3, + 'sdk-examples-api': 3, } diff --git a/botocore/docs/codeexamples.py b/botocore/docs/codeexamples.py index 41f1958d8c..ea4bdf7666 100644 --- a/botocore/docs/codeexamples.py +++ b/botocore/docs/codeexamples.py @@ -10,14 +10,11 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -import os -from botocore import xform_name -from botocore.compat import OrderedDict -from botocore.docs.bcdoc.restdoc import DocumentStructure -from botocore.docs.method import document_model_driven_method -from botocore.docs.utils import DocumentedShape -from botocore.utils import get_service_module_name + +CODE_EXAMPLE_LINK = 'https://docs.aws.amazon.com/code-library/latest/ug/python_3_' +# Change this to the main repo and branch when PR is merged. +CODE_EXAMPLE_CATALOG_BASE = 'https://raw.githubusercontent.com/rlhagerm/aws-doc-sdk-examples/fe9f3a68c94ba888f96e23e799aa8b43cb3f457d' class CodeExamplesDocumenter: @@ -26,13 +23,21 @@ def __init__(self, client, root_docs_path): self._client_class_name = self._client.__class__.__name__ self._service_name = self._client.meta.service_model.service_name self._root_docs_path = root_docs_path - self._USER_GUIDE_LINK = ( - 'https://boto3.amazonaws.com/' - 'v1/documentation/api/latest/guide/clients.html#waiters' - ) - self._CODE_EXAMPLE_LINK = ( - 'https://docs.aws.amazon.com/code-library/latest/ug/python_3_' - ) + + def load_code_examples_catalog(self, loader, service_id): + """Loads the code library example catalog listing for the service. + + :param loader: The loader for the examples catalog. + :param service_id: The code examples service id. + + :return: The list of examples. + """ + git_service_url = f"{CODE_EXAMPLE_CATALOG_BASE}/python/example_code/{service_id}/examples_catalog.json" + examples = loader.file_loader.load_examples_url(git_service_url) + if len(examples) > 0: + return examples['examples'] + + return [] def document_code_examples(self, section, examples, service_id): """Documents the code library code examples for a service. @@ -41,17 +46,16 @@ def document_code_examples(self, section, examples, service_id): :param examples: The list of examples. :param service_id: The code examples service id. """ - section.style.h2('AWS SDK Code Examples') - self._add_overview(section) - - # List the available Code Library examples with a link. + if examples: + section.style.h2('AWS SDK Code Examples') + self._add_overview(section) # Group the examples by category. Do not show a category if there are no examples. example_categories = {} for i in range(len(examples)): example_categories.setdefault(examples[i]['category'], []).append(examples[i]) - # for example in examples: + # Write a link item for each example in the category. for category in example_categories: section.style.new_line() section.style.h3(category) @@ -69,13 +73,17 @@ def document_code_examples(self, section, examples, service_id): section.style.end_li() def _add_overview(self, section): + """Write the overview section for code examples. + + :param section: The section to write to. + """ section.style.new_line() section.write( 'Explore code examples in the ' ) section.style.external_link( title='AWS SDK Code Examples Code Library', - link=self._CODE_EXAMPLE_LINK + self._service_name + '_code_examples.html', + link=CODE_EXAMPLE_LINK + self._service_name + '_code_examples.html', ) section.write('.') section.style.new_line() diff --git a/botocore/docs/service.py b/botocore/docs/service.py index 61afe68581..856659b5b9 100644 --- a/botocore/docs/service.py +++ b/botocore/docs/service.py @@ -46,7 +46,7 @@ def __init__(self, service_name, session, root_docs_path): 'paginator-api', 'waiter-api', 'client-context-params', - 'examples-api', + 'sdk-examples-api', ] def document_service(self): @@ -66,7 +66,7 @@ def document_service(self): 'client-context-params' ) self.client_context_params(context_params_section) - self.examples_api(doc_structure.get_section('examples-api')) + self.sdk_examples_api(doc_structure.get_section('sdk-examples-api')) return doc_structure.flush_structure() def title(self, section): @@ -117,32 +117,20 @@ def waiter_api(self, section): ) waiter_documenter.document_waiters(section) - def examples_api(self, section): + def sdk_examples_api(self, section): library_examples_list = None try: code_example_service_name = SDK_CODE_EXAMPLE_ALIASES.get(self._service_name, self._service_name) - library_examples_list = self.get_library_examples(code_example_service_name) - except DataNotFoundError: - pass - - if library_examples_list: examples_documenter = CodeExamplesDocumenter( self._client, self._root_docs_path ) - examples_documenter.document_code_examples(section, library_examples_list, code_example_service_name) - - def get_library_examples(self, service_name): - loader = self._session.get_component('data_loader') - #TODO: move these constants to config somewhere - git_url = 'https://raw.githubusercontent.com/rlhagerm/aws-doc-sdk-examples/fe9f3a68c94ba888f96e23e799aa8b43cb3f457d/python/example_code/' - git_service_url = f"{git_url}/{service_name}/examples_catalog.json" - - examples = loader.file_loader.load_examples_url(git_service_url) - - if len(examples) > 0: - return examples['examples'] - - return [] + loader = self._session.get_component('data_loader') + library_examples_list = examples_documenter.load_code_examples_catalog(loader, code_example_service_name) + # Only document the examples if there are any in the list. + if library_examples_list: + examples_documenter.document_code_examples(section, library_examples_list, code_example_service_name) + except DataNotFoundError: + pass def get_examples(self, service_name, api_version=None): loader = self._session.get_component('data_loader') diff --git a/botocore/loaders.py b/botocore/loaders.py index fedcbb4d97..36167f5391 100644 --- a/botocore/loaders.py +++ b/botocore/loaders.py @@ -345,7 +345,6 @@ def list_available_services(self, type_name): if self.file_loader.exists(full_load_path): services.add(service_name) break - print(sorted(services)) return sorted(services) @instance_cache diff --git a/tests/unit/docs/test_sdkcodeexamples.py b/tests/unit/docs/test_sdkcodeexamples.py new file mode 100644 index 0000000000..4ed83a1cd7 --- /dev/null +++ b/tests/unit/docs/test_sdkcodeexamples.py @@ -0,0 +1,99 @@ +# Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +from botocore.docs.codeexamples import CodeExamplesDocumenter +from botocore.compat import json +from tests.unit.docs import BaseDocsTest + + +class TestCodeExamplesDocumenter(BaseDocsTest): + def setUp(self): + super().setUp() + self.add_shape_to_params('Biz', 'String') + self.extra_setup() + + def extra_setup(self): + self.setup_client() + self.examples_documenter = CodeExamplesDocumenter( + client=self.client, + root_docs_path=self.root_services_path, + ) + + def test_document_codeexamples(self): + examples_json = ''' + { + "examples": [ + { + "id": "service1_actions1", + "file": "service1_metadata.yaml", + "languages": [], + "title": "Action1", + "category": "Api", + "doc_filenames": { + "service_pages": { + "service1": "https://docs.aws.amazon.com/code-library/latest/ug/service1_example_service1_Action1_section.html" + }, + "sdk_pages": [] + }, + "synopsis_list": [], + "source_key": null + }, + { + "id": "service1_basics", + "file": "service1_metadata.yaml", + "title": "Learn the Basics", + "category": "Basics", + "doc_filenames": { + "service_pages": { + "service1": "https://docs.aws.amazon.com/code-library/latest/ug/service_example_service1_Basics_section.html" + }, + "sdk_pages": [] + }, + "synopsis_list": [], + "source_key": null + }, + { + "id": "service1_Scenario1", + "file": "service1_metadata.yaml", + "title": "Scenario1", + "category": "Scenarios", + "doc_filenames": { + "service_pages": { + "service1": "https://docs.aws.amazon.com/code-library/latest/ug/service_example_service1_Scenario1_section.html" + }, + "sdk_pages": [] + }, + "synopsis_list": [], + "source_key": null + } + ] + } + ''' + examples = json.loads(examples_json) + self.examples_documenter.document_code_examples(self.doc_structure, examples['examples'], 'service1') + self.assert_contains_line('AWS SDK Code Examples') + self.assert_contains_line( + 'Explore code examples in the `AWS SDK Code Examples Code Library `_') + self.assert_contains_line('Api') + self.assert_contains_line('* `Action1 `_') + self.assert_contains_line('Basics') + self.assert_contains_line( + '* `Learn the Basics `_') + self.assert_contains_line('Scenarios') + self.assert_contains_line( + '* `Scenario1 `_') + + def test_no_empty_examples_section(self): + examples = [] + + self.examples_documenter.document_code_examples(self.doc_structure, examples, 'service1') + self.assert_not_contains_line('AWS SDK Code Examples') From 767f8689e977e44a46a035134bd233ef6eee7437 Mon Sep 17 00:00:00 2001 From: Rachel Hagerman <110480692+rlhagerm@users.noreply.github.com> Date: Thu, 27 Mar 2025 09:48:08 -0500 Subject: [PATCH 06/10] Update tests and json load. --- botocore/docs/codeexamples.py | 17 ++++++++----- botocore/docs/service.py | 3 +-- botocore/loaders.py | 28 -------------------- tests/unit/docs/test_sdkcodeexamples.py | 34 +++++++++++++++++++++++++ 4 files changed, 46 insertions(+), 36 deletions(-) diff --git a/botocore/docs/codeexamples.py b/botocore/docs/codeexamples.py index ea4bdf7666..90eaa6a4d7 100644 --- a/botocore/docs/codeexamples.py +++ b/botocore/docs/codeexamples.py @@ -11,6 +11,8 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. +import requests +from botocore.compat import json CODE_EXAMPLE_LINK = 'https://docs.aws.amazon.com/code-library/latest/ug/python_3_' # Change this to the main repo and branch when PR is merged. @@ -24,18 +26,21 @@ def __init__(self, client, root_docs_path): self._service_name = self._client.meta.service_model.service_name self._root_docs_path = root_docs_path - def load_code_examples_catalog(self, loader, service_id): + def load_code_examples_catalog(self, service_id): """Loads the code library example catalog listing for the service. - :param loader: The loader for the examples catalog. :param service_id: The code examples service id. - :return: The list of examples. """ git_service_url = f"{CODE_EXAMPLE_CATALOG_BASE}/python/example_code/{service_id}/examples_catalog.json" - examples = loader.file_loader.load_examples_url(git_service_url) - if len(examples) > 0: - return examples['examples'] + + response = requests.get(git_service_url) + if response.status_code == 200: + example_json = response.text + if example_json: + examples = json.loads(example_json) + if len(examples) > 0: + return examples['examples'] return [] diff --git a/botocore/docs/service.py b/botocore/docs/service.py index 856659b5b9..6764c81226 100644 --- a/botocore/docs/service.py +++ b/botocore/docs/service.py @@ -124,8 +124,7 @@ def sdk_examples_api(self, section): examples_documenter = CodeExamplesDocumenter( self._client, self._root_docs_path ) - loader = self._session.get_component('data_loader') - library_examples_list = examples_documenter.load_code_examples_catalog(loader, code_example_service_name) + library_examples_list = examples_documenter.load_code_examples_catalog(code_example_service_name) # Only document the examples if there are any in the list. if library_examples_list: examples_documenter.document_code_examples(section, library_examples_list, code_example_service_name) diff --git a/botocore/loaders.py b/botocore/loaders.py index 36167f5391..f5072a3e5f 100644 --- a/botocore/loaders.py +++ b/botocore/loaders.py @@ -104,7 +104,6 @@ import logging import os -import requests from botocore import BOTOCORE_ROOT from botocore.compat import HAS_GZIP, OrderedDict, json @@ -198,33 +197,6 @@ def load_file(self, file_path): return data return None - def load_examples_file(self, file_path): - """Load the examples data if available - - :return: The loaded data if it exists, otherwise None. - - """ - - examples_json = self.load_file(file_path) - if examples_json: - return examples_json - return [] - - def load_examples_url(self, file_url): - """Load the examples data from a url if available - - :return: The loaded data if it exists, otherwise None. - - """ - response = requests.get(file_url) - if response.status_code == 200: - example_json = response.text - if example_json: - examples = json.loads(example_json) - return examples - - return [] - def create_loader(search_path_string=None): """Create a Loader class. diff --git a/tests/unit/docs/test_sdkcodeexamples.py b/tests/unit/docs/test_sdkcodeexamples.py index 4ed83a1cd7..45ad2e7bed 100644 --- a/tests/unit/docs/test_sdkcodeexamples.py +++ b/tests/unit/docs/test_sdkcodeexamples.py @@ -13,6 +13,7 @@ from botocore.docs.codeexamples import CodeExamplesDocumenter from botocore.compat import json from tests.unit.docs import BaseDocsTest +from tests import mock class TestCodeExamplesDocumenter(BaseDocsTest): @@ -28,6 +29,39 @@ def extra_setup(self): root_docs_path=self.root_services_path, ) + def mock_response( + self, + status=200, + content="CONTENT", + json_data=None): + + mock_resp = mock.Mock() + # Aet status code and content. + mock_resp.status_code = status + mock_resp.content = content + # Add json data if provided. + if json_data: + mock_resp.text = json_data + return mock_resp + + def test_load_catalog(self): + mock_json = ''' + { + "examples": [ + { + "id": "service1_actions1" + } + ] + } + ''' + examples_result = json.loads(mock_json) + mock_resp = self.mock_response(status=200, json_data=mock_json) + self.request_patch = mock.patch('requests.get', return_value=mock_resp) + self.request_patch.start() + examples = self.examples_documenter.load_code_examples_catalog('service1') + self.request_patch.stop() + self.assertEqual(examples, examples_result['examples']) + def test_document_codeexamples(self): examples_json = ''' { From 75b93669c3856ada51521311fbdbb82142eb07a6 Mon Sep 17 00:00:00 2001 From: Rachel Hagerman <110480692+rlhagerm@users.noreply.github.com> Date: Mon, 31 Mar 2025 12:05:21 -0500 Subject: [PATCH 07/10] Update to category listing. --- botocore/docs/codeexamples.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/botocore/docs/codeexamples.py b/botocore/docs/codeexamples.py index 90eaa6a4d7..ae7ab73c25 100644 --- a/botocore/docs/codeexamples.py +++ b/botocore/docs/codeexamples.py @@ -61,7 +61,7 @@ def document_code_examples(self, section, examples, service_id): example_categories.setdefault(examples[i]['category'], []).append(examples[i]) # Write a link item for each example in the category. - for category in example_categories: + for category in sorted(example_categories): section.style.new_line() section.style.h3(category) for i in range(len(example_categories[category])): From 5a993a9825673651daf28c0c4e589df5522daf80 Mon Sep 17 00:00:00 2001 From: Rachel Hagerman <110480692+rlhagerm@users.noreply.github.com> Date: Wed, 2 Apr 2025 11:46:27 -0500 Subject: [PATCH 08/10] Update to cookie domain as required by marketting. --- docs/source/_static/js/custom.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/_static/js/custom.js b/docs/source/_static/js/custom.js index af879c8697..c432e16dab 100644 --- a/docs/source/_static/js/custom.js +++ b/docs/source/_static/js/custom.js @@ -72,7 +72,7 @@ function isValidFragment(splitFragment) { } } })(); -// Given a service name, we apply the html classes which indicate a current page to the corresponsing list item. +// Given a service name, we apply the html classes which indicate a current page to the corresponding list item. // Before:
  • ACM
    • // After:
  • ACM
    • function makeServiceLinkCurrent(serviceName) { @@ -199,8 +199,8 @@ function setupKeyboardFriendlyNavigation() { function loadShortbread() { if (typeof AWSCShortbread !== "undefined") { const shortbread = AWSCShortbread({ - // If you're testing in your dev environment, use ".cloudfront.net" for domain, else ".amazonaws.com" - domain: ".amazonaws.com", + // If you're testing in your dev environment, use ".cloudfront.net" for domain, else "botocore.amazonaws.com" + domain: "botocore.amazonaws.com", }); // Check for cookie consent From 9abcea4f685f41d3ec95f55410eafa6f5305c3d7 Mon Sep 17 00:00:00 2001 From: Rachel Hagerman <110480692+rlhagerm@users.noreply.github.com> Date: Wed, 2 Apr 2025 11:49:59 -0500 Subject: [PATCH 09/10] Revert "Update to cookie domain as required by marketting." This reverts commit 5a993a9825673651daf28c0c4e589df5522daf80. --- docs/source/_static/js/custom.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/_static/js/custom.js b/docs/source/_static/js/custom.js index c432e16dab..af879c8697 100644 --- a/docs/source/_static/js/custom.js +++ b/docs/source/_static/js/custom.js @@ -72,7 +72,7 @@ function isValidFragment(splitFragment) { } } })(); -// Given a service name, we apply the html classes which indicate a current page to the corresponding list item. +// Given a service name, we apply the html classes which indicate a current page to the corresponsing list item. // Before:
  • ACM
    • // After:
  • ACM
    • function makeServiceLinkCurrent(serviceName) { @@ -199,8 +199,8 @@ function setupKeyboardFriendlyNavigation() { function loadShortbread() { if (typeof AWSCShortbread !== "undefined") { const shortbread = AWSCShortbread({ - // If you're testing in your dev environment, use ".cloudfront.net" for domain, else "botocore.amazonaws.com" - domain: "botocore.amazonaws.com", + // If you're testing in your dev environment, use ".cloudfront.net" for domain, else ".amazonaws.com" + domain: ".amazonaws.com", }); // Check for cookie consent From 9356289c8282bb48af98c07e73900791a01ec597 Mon Sep 17 00:00:00 2001 From: Rachel Hagerman <110480692+rlhagerm@users.noreply.github.com> Date: Thu, 3 Apr 2025 14:45:14 -0500 Subject: [PATCH 10/10] Update catalog version. --- botocore/docs/codeexamples.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/botocore/docs/codeexamples.py b/botocore/docs/codeexamples.py index ae7ab73c25..2ceb40f5ce 100644 --- a/botocore/docs/codeexamples.py +++ b/botocore/docs/codeexamples.py @@ -15,8 +15,9 @@ from botocore.compat import json CODE_EXAMPLE_LINK = 'https://docs.aws.amazon.com/code-library/latest/ug/python_3_' +CODE_EXAMPLE_CATALOG_VERSION = '2025.0.0-alpha' # Change this to the main repo and branch when PR is merged. -CODE_EXAMPLE_CATALOG_BASE = 'https://raw.githubusercontent.com/rlhagerm/aws-doc-sdk-examples/fe9f3a68c94ba888f96e23e799aa8b43cb3f457d' +CODE_EXAMPLE_CATALOG_BASE = 'https://raw.githubusercontent.com/rlhagerm/aws-doc-sdk-examples/refs/tags/' class CodeExamplesDocumenter: @@ -32,7 +33,7 @@ def load_code_examples_catalog(self, service_id): :param service_id: The code examples service id. :return: The list of examples. """ - git_service_url = f"{CODE_EXAMPLE_CATALOG_BASE}/python/example_code/{service_id}/examples_catalog.json" + git_service_url = f"{CODE_EXAMPLE_CATALOG_BASE}/{CODE_EXAMPLE_CATALOG_VERSION}/python/example_code/{service_id}/examples_catalog.json" response = requests.get(git_service_url) if response.status_code == 200: