diff --git a/_data/migration-scenarios.yml b/_data/migration-scenarios.yml index 0bdb919b15..6e808345a7 100644 --- a/_data/migration-scenarios.yml +++ b/_data/migration-scenarios.yml @@ -2,108 +2,108 @@ scenarios: - title: "Scenario 1: Backfill only" steps: - title: "Assessment" - url: "/migration-assistant/migration-phases/assessment/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/assessment/" - title: "Deploy" - url: "/migration-assistant/migration-phases/deploy/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/deploy/" substeps: - - title: "Verify backfill components" - url: "/migration-assistant/migration-phases/deploy/verifying-backfill-components/" - - title: "Configuration options" - url: "/migration-assistant/migration-phases/deploy/configuration-options/" - title: "IAM and security groups for existing clusters" - url: "/migration-assistant/migration-phases/deploy/iam-and-security-groups-for-existing-clusters/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/deploy/iam-and-security-groups-for-existing-clusters/" + - title: "Configuration options" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/deploy/configuration-options/" + - title: "Verify backfill components" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/deploy/verifying-backfill-components/" - title: "Create a snapshot" - url: "/migration-assistant/migration-phases/create-snapshot/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/create-snapshot/" - title: "Migrate metadata" - url: "/migration-assistant/migration-phases/migrate-metadata/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/migrate-metadata/" substeps: - title: "Managing type mapping deprecation" - url: "/migration-assistant/migration-phases/migrate-metadata/handling-type-mapping-deprecation/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/migrate-metadata/handling-type-mapping-deprecation/" - title: "Handling breaking changes in field types" - url: "/migration-assistant/migration-phases/migrate-metadata/handling-field-type-breaking-changes/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/migrate-metadata/handling-field-type-breaking-changes/" - title: "Transform flattened to flat_object" - url: "/migration-assistant/migration-phases/migrate-metadata/transform-flattened-flat-object/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/migrate-metadata/transform-flattened-flat-object/" - title: "Transform string to text/keyword" - url: "/migration-assistant/migration-phases/migrate-metadata/transform-string-text-keyword/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/migrate-metadata/transform-string-text-keyword/" - title: "Transform dense_vector to knn_vector" - url: "/migration-assistant/migration-phases/migrate-metadata/transform-dense-vector-knn-vector/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/migrate-metadata/transform-dense-vector-knn-vector/" - title: "Backfill" - url: "/migration-assistant/migration-phases/backfill/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/backfill/" - title: "Teardown" - url: "/migration-assistant/migration-phases/remove-migration-infrastructure/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/remove-migration-infrastructure/" - title: "Scenario 2: Live capture only" steps: - title: "Assessment" - url: "/migration-assistant/migration-phases/assessment/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/assessment/" - title: "Deploy" - url: "/migration-assistant/migration-phases/deploy/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/deploy/" substeps: - title: "IAM and security groups for existing clusters" - url: "/migration-assistant/migration-phases/deploy/iam-and-security-groups-for-existing-clusters/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/deploy/iam-and-security-groups-for-existing-clusters/" - title: "Configuration options" - url: "/migration-assistant/migration-phases/deploy/configuration-options/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/deploy/configuration-options/" - title: "Verify live capture components" - url: "/migration-assistant/migration-phases/deploy/verifying-live-capture-components/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/deploy/verifying-live-capture-components/" - title: "Reroute traffic from the source to the Capture Proxy" - url: "/migration-assistant/migration-phases/reroute-source-to-proxy/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/reroute-source-to-proxy/" - title: "Migrate metadata" - url: "/migration-assistant/migration-phases/migrate-metadata/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/migrate-metadata/" substeps: - title: "Managing type mapping deprecation" - url: "/migration-assistant/migration-phases/migrate-metadata/handling-type-mapping-deprecation/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/migrate-metadata/handling-type-mapping-deprecation/" - title: "Handling breaking changes in field types" - url: "/migration-assistant/migration-phases/migrate-metadata/handling-field-type-breaking-changes/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/migrate-metadata/handling-field-type-breaking-changes/" - title: "Transform flattened to flat_object" - url: "/migration-assistant/migration-phases/migrate-metadata/transform-flattened-flat-object/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/migrate-metadata/transform-flattened-flat-object/" - title: "Transform string to text/keyword" - url: "/migration-assistant/migration-phases/migrate-metadata/transform-string-text-keyword/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/migrate-metadata/transform-string-text-keyword/" - title: "Transform dense_vector to knn_vector" - url: "/migration-assistant/migration-phases/migrate-metadata/transform-dense-vector-knn-vector/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/migrate-metadata/transform-dense-vector-knn-vector/" - title: "Replay captured traffic" - url: "/migration-assistant/migration-phases/replay-captured-traffic/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/replay-captured-traffic/" - title: "Reroute traffic from the Capture Proxy to the target" - url: "/migration-assistant/migration-phases/reroute-traffic-from-capture-proxy-to-target/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/reroute-traffic-from-capture-proxy-to-target/" - title: "Teardown" - url: "/migration-assistant/migration-phases/remove-migration-infrastructure/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/remove-migration-infrastructure/" - title: "Scenario 3: Live capture with backfill" steps: - title: "Assessment" - url: "/migration-assistant/migration-phases/assessment/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/assessment/" - title: "Deploy" - url: "/migration-assistant/migration-phases/deploy/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/deploy/" substeps: - title: "IAM and security groups for existing clusters" - url: "/migration-assistant/migration-phases/deploy/iam-and-security-groups-for-existing-clusters/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/deploy/iam-and-security-groups-for-existing-clusters/" - title: "Configuration options" - url: "/migration-assistant/migration-phases/deploy/configuration-options/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/deploy/configuration-options/" - title: "Verify backfill components" - url: "/migration-assistant/migration-phases/deploy/verifying-backfill-components/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/deploy/verifying-backfill-components/" - title: "Verify live capture components" - url: "/migration-assistant/migration-phases/deploy/verifying-live-capture-components/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/deploy/verifying-live-capture-components/" - title: "Reroute traffic from the source to the Capture Proxy" - url: "/migration-assistant/migration-phases/reroute-source-to-proxy/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/reroute-source-to-proxy/" - title: "Create a snapshot" - url: "/migration-assistant/migration-phases/create-snapshot/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/create-snapshot/" - title: "Migrate metadata" - url: "/migration-assistant/migration-phases/migrate-metadata/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/migrate-metadata/" substeps: - title: "Managing type mapping deprecation" - url: "/migration-assistant/migration-phases/migrate-metadata/handling-type-mapping-deprecation/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/migrate-metadata/handling-type-mapping-deprecation/" - title: "Handling breaking changes in field types" - url: "/migration-assistant/migration-phases/migrate-metadata/handling-field-type-breaking-changes/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/migrate-metadata/handling-field-type-breaking-changes/" - title: "Transform flattened to flat_object" - url: "/migration-assistant/migration-phases/migrate-metadata/transform-flattened-flat-object/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/migrate-metadata/transform-flattened-flat-object/" - title: "Transform string to text/keyword" - url: "/migration-assistant/migration-phases/migrate-metadata/transform-string-text-keyword/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/migrate-metadata/transform-string-text-keyword/" - title: "Transform dense_vector to knn_vector" - url: "/migration-assistant/migration-phases/migrate-metadata/transform-dense-vector-knn-vector/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/migrate-metadata/transform-dense-vector-knn-vector/" - title: "Backfill" - url: "/migration-assistant/migration-phases/backfill/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/backfill/" - title: "Replay captured traffic" - url: "/migration-assistant/migration-phases/replay-captured-traffic/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/replay-captured-traffic/" - title: "Reroute traffic from the Capture Proxy to the target" - url: "/migration-assistant/migration-phases/reroute-traffic-from-capture-proxy-to-target/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/reroute-traffic-from-capture-proxy-to-target/" - title: "Teardown" - url: "/migration-assistant/migration-phases/remove-migration-infrastructure/" + url: "{{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/remove-migration-infrastructure/" diff --git a/_includes/migration-phase-navigation.html b/_includes/migration-phase-navigation.html new file mode 100644 index 0000000000..f7588b0722 --- /dev/null +++ b/_includes/migration-phase-navigation.html @@ -0,0 +1,193 @@ +{%- comment -%} +This include generates Previous Phase and Next Phase sections for migration phase pages. +It uses the migration scenarios data to determine the flow and relationships. +{%- endcomment -%} + +{%- assign current_page_url = page.url | remove: '/index' -%} +{%- assign scenarios = site.data.migration-scenarios.scenarios -%} + +{%- comment -%}Find previous phases for each scenario{%- endcomment -%} +{%- assign prev_phase_info = '' | split: '' -%} + +{%- for scenario in scenarios -%} + {%- assign found_current = false -%} + {%- assign step_number = 0 -%} + {%- assign prev_step = null -%} + {%- assign prev_step_number = 0 -%} + + {%- for step in scenario.steps -%} + {%- assign step_number = step_number | plus: 1 -%} + {%- assign clean_step_url = step.url | replace: '{{site.url}}{{site.baseurl}}', '' -%} + + {%- if clean_step_url == current_page_url -%} + {%- assign found_current = true -%} + {%- if prev_step -%} + {%- capture prev_phase_data -%}{{ prev_step.url }}|{{ prev_step.title }}|{{ scenario.title }}|{{ prev_step_number }}{%- endcapture -%} + {%- assign prev_phase_info = prev_phase_info | push: prev_phase_data -%} + {%- endif -%} + {%- break -%} + {%- endif -%} + + {%- comment -%}Check substeps too{%- endcomment -%} + {%- if step.substeps -%} + {%- for substep in step.substeps -%} + {%- assign clean_substep_url = substep.url | replace: '{{site.url}}{{site.baseurl}}', '' -%} + {%- if clean_substep_url == current_page_url -%} + {%- assign found_current = true -%} + {%- capture prev_phase_data -%}{{ step.url }}|{{ step.title }}|{{ scenario.title }}|{{ step_number }}{%- endcapture -%} + {%- assign prev_phase_info = prev_phase_info | push: prev_phase_data -%} + {%- break -%} + {%- endif -%} + {%- endfor -%} + {%- if found_current -%} + {%- break -%} + {%- endif -%} + {%- endif -%} + + {%- assign prev_step = step -%} + {%- assign prev_step_number = step_number -%} + {%- endfor -%} +{%- endfor -%} + +{%- comment -%}Remove duplicates from prev_phase_info{%- endcomment -%} +{%- assign unique_prev_phases = '' | split: '' -%} +{%- for phase_data in prev_phase_info -%} + {%- assign phase_parts = phase_data | split: '|' -%} + {%- assign phase_url = phase_parts[0] -%} + {%- assign is_duplicate = false -%} + + {%- for unique_phase_data in unique_prev_phases -%} + {%- assign unique_parts = unique_phase_data | split: '|' -%} + {%- if unique_parts[0] == phase_url -%} + {%- assign is_duplicate = true -%} + {%- break -%} + {%- endif -%} + {%- endfor -%} + + {%- unless is_duplicate -%} + {%- assign unique_prev_phases = unique_prev_phases | push: phase_data -%} + {%- endunless -%} +{%- endfor -%} + +{%- comment -%}Find next phases for each scenario{%- endcomment -%} +{%- assign next_phase_info = '' | split: '' -%} + +{%- for scenario in scenarios -%} + {%- assign step_number = 0 -%} + {%- assign current_step_index = -1 -%} + + {%- comment -%}Find the index of the current step in this scenario{%- endcomment -%} + {%- for step in scenario.steps -%} + {%- assign step_number = step_number | plus: 1 -%} + {%- assign clean_step_url = step.url | replace: '{{site.url}}{{site.baseurl}}', '' -%} + + {%- comment -%}Check if this step matches the current page{%- endcomment -%} + {%- if clean_step_url == current_page_url -%} + {%- assign current_step_index = step_number -%} + {%- endif -%} + + {%- comment -%}Also check substeps{%- endcomment -%} + {%- if step.substeps -%} + {%- for substep in step.substeps -%} + {%- assign clean_substep_url = substep.url | replace: '{{site.url}}{{site.baseurl}}', '' -%} + {%- if clean_substep_url == current_page_url -%} + {%- assign current_step_index = step_number -%} + {%- endif -%} + {%- endfor -%} + {%- endif -%} + {%- endfor -%} + + {%- comment -%}If we found the current step in this scenario, find the next step{%- endcomment -%} + {%- if current_step_index > 0 -%} + {%- assign next_step_index = current_step_index | plus: 1 -%} + {%- assign step_number = 0 -%} + + {%- for step in scenario.steps -%} + {%- assign step_number = step_number | plus: 1 -%} + {%- if step_number == next_step_index -%} + {%- capture next_phase_data -%}{{ step.url }}|{{ step.title }}|{{ scenario.title }}|{{ step_number }}{%- endcapture -%} + {%- assign next_phase_info = next_phase_info | push: next_phase_data -%} + {%- break -%} + {%- endif -%} + {%- endfor -%} + {%- endif -%} +{%- endfor -%} + +{%- comment -%}Remove duplicates from next_phase_info{%- endcomment -%} +{%- assign unique_next_phases = '' | split: '' -%} +{%- for phase_data in next_phase_info -%} + {%- assign phase_parts = phase_data | split: '|' -%} + {%- assign phase_url = phase_parts[0] -%} + {%- assign scenario_title = phase_parts[2] -%} + {%- assign is_duplicate = false -%} + + {%- for unique_phase_data in unique_next_phases -%} + {%- assign unique_parts = unique_phase_data | split: '|' -%} + {%- if unique_parts[0] == phase_url and unique_parts[2] == scenario_title -%} + {%- assign is_duplicate = true -%} + {%- break -%} + {%- endif -%} + {%- endfor -%} + + {%- unless is_duplicate -%} + {%- assign unique_next_phases = unique_next_phases | push: phase_data -%} + {%- endunless -%} +{%- endfor -%} + + +{%- comment -%}Display Previous Phase section{%- endcomment -%} +{%- if unique_prev_phases.size > 0 %} + +## Previous phase + +{%- if unique_prev_phases.size == 1 -%} +{%- assign phase_data = unique_prev_phases[0] -%} +{%- assign phase_parts = phase_data | split: '|' %} +{%- assign clean_prev_url = phase_parts[0] | replace: '{{site.url}}{{site.baseurl}}', '' %} + +The previous phase in the migration process was: + +**[{{ phase_parts[1] }}]({{ site.url }}{{ site.baseurl }}{{ clean_prev_url }})** + +{%- else %} + +The previous phase depends on your migration scenario: + +{% for phase_data in unique_prev_phases -%} +{%- assign phase_parts = phase_data | split: '|' %} +{%- assign clean_prev_url = phase_parts[0] | replace: '{{site.url}}{{site.baseurl}}', '' -%} +- **{{ phase_parts[2] }}** (Step {{ phase_parts[3] }}): [{{ phase_parts[1] }}]({{ site.url }}{{ site.baseurl }}{{ clean_prev_url }}) +{% endfor %} +{%- endif %} +{%- endif %} + +{%- comment -%}Display Next Phase section{%- endcomment -%} +{%- if unique_next_phases.size > 0 %} + +## Next phase + +{%- if unique_next_phases.size == 1 -%} +{%- assign phase_data = unique_next_phases[0] -%} +{%- assign phase_parts = phase_data | split: '|' %} +{%- assign clean_next_url = phase_parts[0] | replace: '{{site.url}}{{site.baseurl}}', '' %} + +The next phase in the migration process is: + +**[{{ phase_parts[1] }}]({{ site.url }}{{ site.baseurl }}{{ clean_next_url }})** + +For a complete overview of all migration phases, see [Migration phases]({{ site.url }}{{ site.baseurl }}/migration-assistant/migration-phases/). + +{%- else %} + +The next phase depends on your migration scenario: + +{% for phase_data in unique_next_phases -%} +{%- assign phase_parts = phase_data | split: '|' %} +{%- assign clean_next_url = phase_parts[0] | replace: '{{site.url}}{{site.baseurl}}', '' -%} +- **{{ phase_parts[2] }}** (Step {{ phase_parts[3] }}): [{{ phase_parts[1] }}]({{ site.url }}{{ site.baseurl }}{{ clean_next_url }}) +{% endfor %} + +For a complete overview of all migration phases, see [Migration phases]({{ site.url }}{{ site.baseurl }}/migration-assistant/migration-phases/). + +{%- endif %} +{%- endif %} diff --git a/_includes/migration-scenarios.html b/_includes/migration-scenarios.html index cdb02587c8..e25665a9ed 100644 --- a/_includes/migration-scenarios.html +++ b/_includes/migration-scenarios.html @@ -8,7 +8,8 @@ {% for step in scenario.steps %}
  • {% if step.url %} - {{ step.title }} + {% assign clean_step_url = step.url | replace: '{{site.url}}{{site.baseurl}}', '' %} + {{ step.title }} {% else %} {{ step.title }} {% endif %} @@ -18,7 +19,8 @@ {% for substep in step.substeps %}
  • {% if substep.url %} - {{ substep.title }} + {% assign clean_substep_url = substep.url | replace: '{{site.url}}{{site.baseurl}}', '' %} + {{ substep.title }} {% else %} {{ substep.title }} {% endif %} diff --git a/_migration-assistant/index.md b/_migration-assistant/index.md index 666788ffc1..d714feb56a 100644 --- a/_migration-assistant/index.md +++ b/_migration-assistant/index.md @@ -24,13 +24,12 @@ items: # Migration Assistant for OpenSearch -Migration Assistant for OpenSearch helps you successfully perform an end-to-end, zero-downtime upgrade and migration to OpenSearch using the following steps: +Migration Assistant for OpenSearch helps you successfully perform an end-to-end, zero-downtime upgrade and migration to OpenSearch. There are three aspects of a migration that must be understood: - **Metadata migration**: Migrate cluster metadata, such as index settings, aliases, and templates. - **Backfill migration**: Migrate existing or historical data from a source to a target cluster. - **Live traffic migration**: Replicate live ongoing traffic from a source to a target cluster. -- **Comparative tooling**: Compare the performance and behaviors of an existing cluster with a prospective new one. -This user guide focuses on conducting a comprehensive migration involving both existing and live data with zero downtime and the option to back out of a migration. +This user guide focuses on conducting a comprehensive migration. {% include list.html list_items=page.items%} diff --git a/_migration-assistant/is-migration-assistant-right-for-you.md b/_migration-assistant/is-migration-assistant-right-for-you.md index 63b2977c3e..7c4ea67c77 100644 --- a/_migration-assistant/is-migration-assistant-right-for-you.md +++ b/_migration-assistant/is-migration-assistant-right-for-you.md @@ -88,7 +88,6 @@ Use this checklist to determine whether Migration Assistant is the right fit for - Are you upgrading but want the ability to safely back out, reducing the risk of data loss or service disruption? - Do you need to maintain high service availability with minimal or zero downtime? - Do you need to validate a new OpenSearch cluster before switching over—with rollback capabilities? -- Is your environment self-managed or running on Amazon OpenSearch Service? - Are you looking for tooling to migrate index settings and other metadata? - Do you need to reconfigure your target cluster—for example, by changing the sharding strategy and reindexing? - Are you migrating across Regions, from on premises, or from another cloud provider? diff --git a/_migration-assistant/key-components.md b/_migration-assistant/key-components.md index aaf05668c4..cce18a4618 100644 --- a/_migration-assistant/key-components.md +++ b/_migration-assistant/key-components.md @@ -14,11 +14,11 @@ The following are the key components of Migration Assistant. ## Elasticsearch/OpenSearch source -In this solution, your source cluster operates on either Elasticsearch or OpenSearch and is hosted on Amazon Elastic Compute Cloud (Amazon EC2) instances or in a similar computing environment. Traffic is rerouted from the source cluster to a Traffic Capture Proxy and replayed to a target, typically on a later version of OpenSearch. +In this solution, your source cluster operates on either Elasticsearch or OpenSearch and is hosted on Amazon Elastic Compute Cloud (Amazon EC2) instances or in a similar computing environment. The source cluster may also be managed by AWS, an alternative cloud provider, or hosted on premises. Traffic is rerouted from the source cluster to a Traffic Capture Proxy and replayed to a target, typically on a later version of OpenSearch. ## Migration console -The migration console provides a migration-specific CLI and offers a variety of tools for streamlining the migration process. You can perform everything necessary for completing a migration, other than cleaning up the migration resources, through this console. +The migration console provides a migration-specific CLI and offers a variety of tools for streamlining the migration process. You can perform everything necessary for completing a migration, other than cleaning up the migration resources and managing application changes, through this console. ## Traffic Capture Proxy @@ -38,4 +38,4 @@ The metadata migration tool integrated into the Migration CLI can be used indepe ## Target cluster -The target cluster is the destination cluster for migration or the comparison cluster in an A/B test. +The target cluster is the destination cluster for an upgrade or migration. It may also be a cluster on the same version that has been reconfigured to meet changing application needs. diff --git a/_migration-assistant/migration-phases/assessment/index.md b/_migration-assistant/migration-phases/assessment/index.md index d5829c3585..124a7eac15 100644 --- a/_migration-assistant/migration-phases/assessment/index.md +++ b/_migration-assistant/migration-phases/assessment/index.md @@ -69,3 +69,5 @@ The following is a list of transformations that are included in Migration Assist - [Managing type mapping deprecation]({{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/migrate-metadata/handling-type-mapping-deprecation/) - [Handling breaking changes in field types]({{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/migrate-metadata/handling-field-type-breaking-changes/) + +{% include migration-phase-navigation.html %} diff --git a/_migration-assistant/migration-phases/backfill.md b/_migration-assistant/migration-phases/backfill.md index 8704d8a35d..1f260c09f6 100644 --- a/_migration-assistant/migration-phases/backfill.md +++ b/_migration-assistant/migration-phases/backfill.md @@ -26,20 +26,6 @@ console backfill start ``` {% include copy.html %} -You should receive a response similar to the following: - -```shell -BackfillStatus.RUNNING -Running=1 -Pending=0 -Desired=1 -Shards total: 48 -Shards completed: 48 -Shards incomplete: 0 -Shards in progress: 0 -Shards unclaimed: 0 -``` - The status will be `Running` even if all the shards have been migrated. ### Scaling up the fleet @@ -179,3 +165,5 @@ fields @message {% include copy.html %} If any failed documents are identified, you can index the failed documents directly as opposed to using RFS. + +{% include migration-phase-navigation.html %} diff --git a/_migration-assistant/migration-phases/create-snapshot.md b/_migration-assistant/migration-phases/create-snapshot.md index 9cc80ad9dd..09d165a19c 100644 --- a/_migration-assistant/migration-phases/create-snapshot.md +++ b/_migration-assistant/migration-phases/create-snapshot.md @@ -9,7 +9,7 @@ permalink: /migration-assistant/migration-phases/create-snapshot/ # Creating a snapshot -Creating a snapshot of the source cluster captures all the metadata and documents to be migrated to a new target cluster. +Once you have your change data capture solution in place or have disabled indexing to your source cluster, you are ready to create a snapshot. Creating a snapshot of the source cluster captures all the metadata and documents to be migrated to a new target cluster. ## Create a snapshot @@ -20,13 +20,9 @@ console snapshot create [...] ``` {% include copy.html %} -**Note**: Migration Assistant will automatically generate a snapshot name and configure the necessary Amazon Simple Storage Service (Amazon S3) repository. You only need to specify `--snapshot-name` if you are using an existing snapshot that you created outside of Migration Assistant. +**Note**: Migration Assistant will automatically generate a snapshot name and configure the necessary Amazon Simple Storage Service (Amazon S3) repository. Alternatively, you have the option to bring your own existing snapshot. -**For existing snapshots**: -If you have an existing snapshot, you can specify it during the RFS configuration in your `cdk.context.json` file using the `reindexFromSnapshotExtraArgs` parameter: -```json -"reindexFromSnapshotExtraArgs": "--s3-repo-uri s3://your-bucket/repo --s3-region us-west-2 --snapshot-name your-existing-snapshot" -``` +For more information about using an existing snapshot, see [Bring your own snapshot]({{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/deploy/configuration-options/#bring-your-own-snapshot) configuration. To check the snapshot creation status, run the following command: @@ -42,14 +38,7 @@ console snapshot status --deep-check ``` {% include copy.html %} -Wait for snapshot creation to complete before proceeding to the next migration phase. - -To check the progress of the snapshot in real time, use the following command: - -```shell -console snapshot status --deep-check -``` -{% include copy.html %} +Wait for snapshot creation to complete before proceeding to the metadata migration phase. You should receive the following response when the snapshot is created: @@ -70,3 +59,5 @@ Throughput: 38.13 MiB/sec ## Managing slow snapshot speeds Depending on the size of the data in the source cluster and the bandwidth allocated for snapshots, the process can take some time. Adjust the maximum rate at which the source cluster's nodes create the snapshot using the `--max-snapshot-rate-mb-per-node` option. Increasing the snapshot rate will consume more node resources, which may affect the cluster's ability to handle normal traffic. + +{% include migration-phase-navigation.html %} diff --git a/_migration-assistant/migration-phases/deploy/configuration-options.md b/_migration-assistant/migration-phases/deploy/configuration-options.md index 7194b0d33d..9fe37d5780 100644 --- a/_migration-assistant/migration-phases/deploy/configuration-options.md +++ b/_migration-assistant/migration-phases/deploy/configuration-options.md @@ -253,3 +253,5 @@ If the S3 bucket is in a separate AWS account from the Migration Assistant deplo ## Network configuration The migration tooling expects the source cluster, target cluster, and migration resources to exist in the same VPC. If this is not the case, manual networking setup outside of this documentation is likely required. + +{% include migration-phase-navigation.html %} diff --git a/_migration-assistant/migration-phases/deploy/iam-and-security-groups-for-existing-clusters.md b/_migration-assistant/migration-phases/deploy/iam-and-security-groups-for-existing-clusters.md index ad7e30e0bf..40300a0082 100644 --- a/_migration-assistant/migration-phases/deploy/iam-and-security-groups-for-existing-clusters.md +++ b/_migration-assistant/migration-phases/deploy/iam-and-security-groups-for-existing-clusters.md @@ -14,9 +14,9 @@ redirect_from: This page outlines security scenarios for using the migration tools with existing clusters, including any necessary configuration changes to ensure proper communication between them. -## Importing an Amazon OpenSearch Service or Amazon OpenSearch Serverless target cluster +## Importing Amazon OpenSearch Service -Use the following scenarios for Amazon OpenSearch Service or Amazon OpenSearch Serverless target clusters. +Use the following configuration for Amazon OpenSearch Service. ### OpenSearch Service @@ -41,57 +41,13 @@ Use the following steps to grant the required permissions: 3. Go to the **Resources** tab and locate the following IAM roles: ```bash - arn:aws:iam::****:role/OSMigrations---MigrationServiceTaskRoleC- - arn:aws:iam::****:role/OSMigrations---reindexfromsnapshotTaskRo- - arn:aws:iam::****:role/OSMigrations---trafficreplayerdefaultTas- + arn:aws:iam::****:role/OSMigrations---migration-console-task + arn:aws:iam::****:role/OSMigrations---reindex-from-snapshot-task + arn:aws:iam::****:role/OSMigrations---traffic-replayer-default-task ``` 4. In both the source and target clusters, map users to each Amazon Resource Name (ARN) using the following steps: A. Access OpenSearch Dashboards. If you're using Elasticsearch, access Kibana. B. Navigate to **Security -> Roles -> all_access**. C. In the "Mapped users" section, add each ARN as a backend role. - D. Save your changes. - -### OpenSearch Serverless - -For an OpenSearch Serverless Collection, you will need to configure both network and data access policies: - -1. **Network Policy Configuration**: - The Collection should have a network policy that uses the `VPC` access type. This requires creating a VPC endpoint on the VPC used for the solution. The VPC endpoint should be configured for the private subnets of the VPC and should attach the `osClusterAccessSG` security group. - -2. **Data Access Policy Configuration**: - The data access policy should grant permission to perform all [index operations](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-data-access.html#serverless-data-supported-permissions) (`aoss:*`) for all indexes in the Collection. The IAM task roles of the applicable Migration services (Traffic Replayer, migration console, `Reindex-from-Snapshot`) should be used as the principals for this data access policy. - -## Capture Proxy on Coordinator Nodes of Source Cluster - -Although the CDK does not automatically set up the Capture Proxy on source cluster nodes (except in the demo solution), the Capture Proxy instances must communicate with the resources deployed by the CDK, such as Kafka. This section outlines the necessary steps to set up communication. - -Before [setting up Capture Proxy instances](https://github.com/opensearch-project/opensearch-migrations/tree/main/TrafficCapture/trafficCaptureProxyServer#installing-capture-proxy-on-coordinator-nodes) on the source cluster, ensure the following configurations are in place: - -1. **Security group Configuration**: - The coordinator nodes should add the `trafficStreamSourceSG` security group to allow sending captured traffic to Kafka. - -2. **IAM Policy Configuration**: - The IAM role used by the coordinator nodes should have permissions to publish captured traffic to Kafka. You can add the following template policy through the AWS Console (IAM Role → Add permissions → Create inline policy → JSON view): - -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Action": "kafka-cluster:Connect", - "Resource": "arn:aws:kafka:::cluster/migration-msk-cluster-/*", - "Effect": "Allow" - }, - { - "Action": [ - "kafka-cluster:CreateTopic", - "kafka-cluster:DescribeTopic", - "kafka-cluster:WriteData" - ], - "Resource": "arn:aws:kafka:::topic/migration-msk-cluster-/*", - "Effect": "Allow" - } - ] -} -``` + D. Save your changes. \ No newline at end of file diff --git a/_migration-assistant/migration-phases/deploy/index.md b/_migration-assistant/migration-phases/deploy/index.md index 68bce18af2..72df2f3a1a 100644 --- a/_migration-assistant/migration-phases/deploy/index.md +++ b/_migration-assistant/migration-phases/deploy/index.md @@ -5,6 +5,7 @@ parent: Migration phases grand_parent: Migration Assistant for OpenSearch nav_order: 2 has_children: true +has_toc: true permalink: /migration-assistant/migration-phases/deploy/ redirect_from: - /migration-assistant/getting-started-with-data-migration/ @@ -371,3 +372,5 @@ After completing the deployment, proceed with the migration phases: 3. **[Backfill]({{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/backfill/)**: Migrate documents and monitor the process. For more information about the complete migration process, see [Migration phases]({{site.url}}{{site.baseurl}}/migration-assistant/migration-phases/). + +{% include migration-phase-navigation.html %} diff --git a/_migration-assistant/migration-phases/deploy/verifying-backfill-components.md b/_migration-assistant/migration-phases/deploy/verifying-backfill-components.md index fd65d941b5..966f7ba408 100644 --- a/_migration-assistant/migration-phases/deploy/verifying-backfill-components.md +++ b/_migration-assistant/migration-phases/deploy/verifying-backfill-components.md @@ -110,66 +110,6 @@ Older versions of the Elasticsearch S3 repository plugin may have trouble readin If you encounter this issue, you can resolve it by temporarily enabling IMDSv1 on the instances in your source cluster for the duration of the snapshot. There is a toggle for this available in the AWS Management Console and in the AWS CLI. Switching this toggle will turn on the older access model and enable the Elasticsearch S3 repository plugin to work as normal. For more information about IMDSv1, see [Modify instance metadata options for existing instances](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-IMDS-existing-instances.html). -## Switching over client traffic - -The Migration Assistant Application Load Balancer is deployed with a listener that shifts traffic between the source and target clusters through proxy services. The Application Load Balancer should start in **Source Passthrough** mode. - -### Verifying that the traffic switchover is complete - -Use the following steps to verify that the traffic switchover is complete: - -1. In the AWS Management Console, navigate to **EC2 > Load Balancers**. -2. Select the **MigrationAssistant ALB**. -3. Examine the listener on port `9200` and verify that 100% of the traffic is directed to the **Source Proxy**. -4. Navigate to the **Migration ECS Cluster** in the AWS Management Console. -5. Select the **Target Proxy Service**. -6. Verify that the desired count for the service is running: - * If the desired count is not met, update the service to increase it to at least 1 and wait for the service to start. -7. On the **Health and Metrics** tab under **Load balancer health**, verify that all targets are reporting as healthy: - * This confirms that the Application Load Balancer can connect to the target cluster through the target proxy. -8. (Reset) Update the desired count for the **Target Proxy Service** back to its original value in Amazon ECS. - -### Fixing unidentified traffic patterns - -When switching over traffic to the target cluster, you might encounter unidentified traffic patterns. To help identify the cause of these patterns, use the following steps: -* Verify that the target cluster allows traffic ingress from the **Target Proxy security group**. -* Navigate to **Target Proxy ECS Tasks** to investigate any failing tasks. -Set the **Filter desired status** to **Any desired status** to view all tasks, then navigate to the logs for any stopped tasks. - - -## Verifying replication - -Use the following steps to verify that replication is working once the Traffic Capture Proxy is deployed: - - -1. Navigate to the **Migration ECS Cluster** in the AWS Management Console. -2. Navigate to **Capture Proxy Service**. -3. Verify that the capture proxy is running with the desired proxy count. If it is not, update the service to increase it to at least 1 and wait for startup. -4. Under **Health and Metrics** > **Load balancer health**, verify that all targets are healthy. This means that the Application Load Balancer is able to connect to the source cluster through the capture proxy. -5. Navigate to the **Migration Console Terminal**. -6. Run `console kafka describe-topic-records`. Wait 30 seconds for another Application Load Balancer health check. -7. Run `console kafka describe-topic-records` again and verify that the number of RECORDS increased between runs. -8. Run `console replay start` to start Traffic Replayer. -9. Run `tail -f /shared-logs-output/traffic-replayer-default/*/tuples/tuples.log | jq '.targetResponses[]."Status-Code"'` to confirm that the Kafka requests were sent to the target and that it responded as expected. If the responses don't appear: - * Check that the migration console can access the target cluster by running `./catIndices.sh`, which should show the indexes in the source and target. - * Confirm that messages are still being recorded to Kafka. - * Check for errors in the Traffic Replayer logs (`/migration/STAGE/default/traffic-replayer-default`) using Amazon CloudWatch. -10. (Reset) Update the desired count for the **Capture Proxy Service** back to its original value in Amazon ECS. - -### Troubleshooting - -Use this guidance to troubleshoot any of the following replication verification issues. - -### Health check responses with 401/403 status code - -If the source cluster is configured to require authentication, the capture proxy will not be able to verify replication beyond receiving a 401/403 status code for Application Load Balancer health checks. For more information, see [Failure Modes](https://github.com/opensearch-project/opensearch-migrations/blob/main/TrafficCapture/trafficCaptureProxyServer/README.md#failure-modes). - -### Traffic does not reach the source cluster - -Verify that the source cluster allows traffic ingress from the Capture Proxy security Group. - -Look for failing tasks by navigating to **Traffic Capture Proxy ECS**. Change **Filter desired status** to **Any desired status** in order to see all tasks and navigate to the logs for stopped tasks. - ### Snapshot and S3 bucket issues When using the CDK deployment for Migration Assistant, you might encounter the following errors during snapshot creation and deletion. @@ -183,13 +123,4 @@ To make sure that you can delete snapshots and create them during the AWS Cloud #### Snapshot conflicts -To prevent snapshot conflicts, use the `console snapshot delete` command from the migration console. If you delete snapshots or snapshot repositories in a location other than the migration console, you might encounter "already exists" errors. - -## Resetting before migration - -After all verifications are complete, reset all resources before using Migration Assistant for an actual migration. - -The following steps outline how to reset resources with Migration Assistant before executing the actual migration. At this point all verifications are expected to have been completed. These steps can be performed after [Accessing the Migration Console]({{site.url}}{{site.baseurl}}/migration-assistant/migration-console/accessing-the-migration-console/). - - -{% include copy.html %} +To prevent snapshot conflicts, use the `console snapshot delete` command from the migration console. If you delete snapshots or snapshot repositories in a location other than the migration console, you might encounter "already exists" errors. \ No newline at end of file diff --git a/_migration-assistant/migration-phases/deploy/verifying-live-capture-components.md b/_migration-assistant/migration-phases/deploy/verifying-live-capture-components.md index 6ab27376e8..4b9e7a9e7b 100644 --- a/_migration-assistant/migration-phases/deploy/verifying-live-capture-components.md +++ b/_migration-assistant/migration-phases/deploy/verifying-live-capture-components.md @@ -103,21 +103,6 @@ Verify that the source cluster allows traffic ingress from the Capture Proxy sec Look for failing tasks by navigating to **Traffic Capture Proxy ECS**. Change **Filter desired status** to **Any desired status** in order to see all tasks and navigate to the logs for stopped tasks. -### Snapshot and S3 bucket issues - -When using the AWS Cloud Development Kit (AWS CDK) deployment for Migration Assistant, you might encounter the following errors during snapshot creation and deletion. - -#### Bucket permissions - -To make sure that you can delete snapshots and create them during the CDK deployment process, confirm that the `OSMigrations-dev--CustomS3AutoDeleteObjects` stack has S3 object deletion rights. Then, verify that `OSMigrations-dev--default-SnapshotRole` has the following S3 permissions: - - - List bucket contents - - Read/Write/Delete objects - -#### Snapshot conflicts - -To prevent snapshot conflicts, use the `console snapshot delete` command from the migration console. If you delete snapshots or snapshot repositories in a location other than the migration console, you might encounter "already exists" errors. - ## Resetting before migration After all verifications are complete, reset all resources before using Migration Assistant for an actual migration. diff --git a/_migration-assistant/migration-phases/index.md b/_migration-assistant/migration-phases/index.md index 2800a0226f..dd0197d383 100644 --- a/_migration-assistant/migration-phases/index.md +++ b/_migration-assistant/migration-phases/index.md @@ -16,6 +16,6 @@ redirect_from: # Migration phases -This page outlines the core phases of migrating with Migration Assistant. There are three migration scenarios, each consisting of a sequence of common steps. +This page outlines the phases of migrating with Migration Assistant. There are three migration scenarios, each consisting of a sequence of common steps. {% include migration-scenarios.html scenarios=site.data.migration-scenarios.scenarios %} diff --git a/_migration-assistant/migration-phases/migrate-metadata/index.md b/_migration-assistant/migration-phases/migrate-metadata/index.md index 0286aba2dd..bda5b87e18 100644 --- a/_migration-assistant/migration-phases/migrate-metadata/index.md +++ b/_migration-assistant/migration-phases/migrate-metadata/index.md @@ -5,7 +5,7 @@ nav_order: 5 parent: Migration phases grand_parent: Migration Assistant for OpenSearch has_children: true -has_toc: false +has_toc: true permalink: /migration-assistant/migration-phases/migrate-metadata/ redirect_from: - /migration-assistant/migration-phases/migrating-metadata/ @@ -30,11 +30,6 @@ console metadata evaluate --help ``` {% include copy.html %} -```shell -console metadata migrate --help -``` -{% include copy.html %} - Based on the migration console deployment options, a number of commands will be pre-populated. To view them, run console with verbosity: ```shell @@ -57,7 +52,7 @@ INFO:console_link.models.metadata:Migrating metadata with command: /root/metadat ``` -## Using the `evaluate` command +## Using the evaluate command By scanning the contents of the source cluster, applying filtering, and applying modifications a list of all items that will be migrated will be created. Any items not seen in this output will not be migrated onto the target cluster if the migrate command was to be run. This is a safety check before making modifications on the target cluster. @@ -222,3 +217,5 @@ As Metadata migration supports migrating from ES 6.8 on to the latest versions o {% include copy.html %} For additional technical details, [view the mapping type removal source code](https://github.com/opensearch-project/opensearch-migrations/blob/main/transformation/src/main/java/org/opensearch/migrations/transformation/rules/IndexMappingTypeRemoval.java). + +{% include migration-phase-navigation.html %} diff --git a/_migration-assistant/migration-phases/remove-migration-infrastructure.md b/_migration-assistant/migration-phases/remove-migration-infrastructure.md index 239bb346c5..448570fc63 100644 --- a/_migration-assistant/migration-phases/remove-migration-infrastructure.md +++ b/_migration-assistant/migration-phases/remove-migration-infrastructure.md @@ -1,7 +1,7 @@ --- layout: default title: Removing Migration Assistant -nav_order: 8 +nav_order: 9 parent: Migration phases grand_parent: Migration Assistant for OpenSearch permalink: /migration-assistant/migration-phases/remove-migration-infrastructure/ @@ -30,3 +30,5 @@ You can also use the AWS Management Console to remove Migration Assistant resour You can uninstall Migration Assistant for OpenSearch Service from the AWS Management Console or by using the AWS Command Line Interface (AWS CLI). Manually remove the contents of the Amazon Simple Storage Service (Amazon S3) bucket that matches the syntax `cdk--assets--`, the bucket created by Migration Assistant. Migration Assistant for OpenSearch does not automatically delete S3 buckets. To delete the stored data and the AWS CloudFormation stacks created by Migration Assistant, see [Uninstall the solution](https://docs.aws.amazon.com/solutions/latest/migration-assistant-for-amazon-opensearch-service/uninstall-the-solution.html) in the Amazon OpenSearch Service documentation. + +{% include migration-phase-navigation.html %} diff --git a/_migration-assistant/migration-phases/replay-captured-traffic.md b/_migration-assistant/migration-phases/replay-captured-traffic.md index f961ea80df..8f28f2e834 100644 --- a/_migration-assistant/migration-phases/replay-captured-traffic.md +++ b/_migration-assistant/migration-phases/replay-captured-traffic.md @@ -12,7 +12,7 @@ redirect_from: # Using Traffic Replayer -**Note**: This page is only relevant if you are using Capture and Replay to avoid downtime during a migration. If you are only performing backfill migration or can tolerate downtime, you can skip this step. +**Note**: This page is only relevant if you are using Capture and Replay to avoid downtime during a migration. If you are only performing backfill migration, you can skip this step. {: .note} This guide covers how to use Traffic Replayer to replay captured traffic from a source cluster to a target cluster during the migration process. Traffic Replayer allows you to verify that the target cluster can handle requests in the same way as the source cluster and catch up to real-time traffic for a smooth migration. @@ -322,3 +322,5 @@ The following metrics are also reported: ## CloudWatch considerations Metrics and dashboards pushed to CloudWatch may experience a visibility lag of around 5 minutes. CloudWatch also retains higher-resolution data for a shorter period than lower-resolution data. For more information, see [Amazon CloudWatch concepts](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/cloudwatch_concepts.html). + +{% include migration-phase-navigation.html %} diff --git a/_migration-assistant/migration-phases/reroute-source-to-proxy.md b/_migration-assistant/migration-phases/reroute-source-to-proxy.md index ff01b494b7..4cb6d03376 100644 --- a/_migration-assistant/migration-phases/reroute-source-to-proxy.md +++ b/_migration-assistant/migration-phases/reroute-source-to-proxy.md @@ -9,7 +9,7 @@ permalink: /migration-assistant/migration-phases/reroute-source-to-proxy/ # Reroute client traffic to the Traffic Capture Proxy -**Note**: This page is only relevant if you are using Capture and Replay to avoid downtime during a migration. If you are only performing backfill migration or can tolerate downtime, you can skip this step. +**Note**: This page is only relevant if you are using Capture and Replay to avoid downtime during a migration. If you are only performing backfill migration, you can skip this step. {: .note} ## Capture Proxy data replication @@ -87,3 +87,5 @@ TARGET CLUSTER health status index uuid pri rep docs.count docs.deleted store.size pri.store.size green open .opendistro_security N3uy88FGT9eAO7FTbLqqqA 1 0 10 0 78.3kb 78.3kb ``` + +{% include migration-phase-navigation.html %} diff --git a/_migration-assistant/migration-phases/reroute-traffic-from-capture-proxy-to-target.md b/_migration-assistant/migration-phases/reroute-traffic-from-capture-proxy-to-target.md index 02a6d0aec0..e11e098c1d 100644 --- a/_migration-assistant/migration-phases/reroute-traffic-from-capture-proxy-to-target.md +++ b/_migration-assistant/migration-phases/reroute-traffic-from-capture-proxy-to-target.md @@ -1,7 +1,7 @@ --- layout: default title: Reroute traffic to the target -nav_order: 7 +nav_order: 8 parent: Migration phases grand_parent: Migration Assistant for OpenSearch permalink: /migration-assistant/migration-phases/reroute-traffic-from-capture-proxy-to-target/ @@ -9,7 +9,7 @@ permalink: /migration-assistant/migration-phases/reroute-traffic-from-capture-pr # Switching traffic from the source cluster -**Note**: This page is only relevant if you are using Capture and Replay to avoid downtime during a migration. If you are only performing backfill migration or can tolerate downtime, you can skip this step. +**Note**: This page is only relevant if you are using Capture and Replay to avoid downtime during a migration. If you are only performing backfill migration, you can skip this step. {: .note} After the source and target clusters are synchronized, traffic needs to be switched to the target cluster so that the source cluster can be taken offline. @@ -55,3 +55,5 @@ Use the following steps to switch traffic from the source cluster to the target ## Fallback If you need to fall back to the source cluster at any point during the switchover, revert the **Default rule** so that the Application Load Balancer routes to the **SourceProxy Target Group**. + +{% include migration-phase-navigation.html %}