diff --git a/doc/reference/internals/iproto/images/repl_fetch_snapshot_request.puml b/doc/reference/internals/iproto/images/repl_fetch_snapshot_request.puml new file mode 100644 index 000000000..5779ea724 --- /dev/null +++ b/doc/reference/internals/iproto/images/repl_fetch_snapshot_request.puml @@ -0,0 +1,18 @@ +@startuml + +skinparam { + HyperlinkColor #0077FF + FontColor #313131 + BorderColor #313131 + BackgroundColor transparent +} + +json "**IPROTO_FETCH_SNAPSHOT**" as fetch_snapshot_request { + "Size": "MP_UINT", + "Header": { + "[[https://tarantool.io/en/doc/latest/reference/internals/iproto/keys IPROTO_REQUEST_TYPE]]": "IPROTO_FETCH_SNAPSHOT", + "[[https://tarantool.io/en/doc/latest/reference/internals/iproto/keys IPROTO_SYNC]]": "MP_UINT" + } +} + +@enduml diff --git a/doc/reference/internals/iproto/images/repl_fetch_snapshot_request.svg b/doc/reference/internals/iproto/images/repl_fetch_snapshot_request.svg new file mode 100644 index 000000000..240afb874 --- /dev/null +++ b/doc/reference/internals/iproto/images/repl_fetch_snapshot_request.svg @@ -0,0 +1,28 @@ +IPROTO_FETCH_SNAPSHOTSizeMP_UINTHeaderIPROTO_REQUEST_TYPEIPROTO_FETCH_SNAPSHOTIPROTO_SYNCMP_UINT \ No newline at end of file diff --git a/doc/reference/internals/iproto/images/repl_join_meta.puml b/doc/reference/internals/iproto/images/repl_join_meta.puml new file mode 100644 index 000000000..23ce974da --- /dev/null +++ b/doc/reference/internals/iproto/images/repl_join_meta.puml @@ -0,0 +1,18 @@ +@startuml + +skinparam { + HyperlinkColor #0077FF + FontColor #313131 + BorderColor #313131 + BackgroundColor transparent +} + +json "**IPROTO_JOIN_META**" as join_meta_request { + "Size": "MP_UINT", + "Header": { + "[[https://tarantool.io/en/doc/latest/reference/internals/iproto/keys IPROTO_REQUEST_TYPE]]": "IPROTO_JOIN_META", + "[[https://tarantool.io/en/doc/latest/reference/internals/iproto/keys IPROTO_SYNC]]": "MP_UINT" + } +} + +@enduml diff --git a/doc/reference/internals/iproto/images/repl_join_meta.svg b/doc/reference/internals/iproto/images/repl_join_meta.svg new file mode 100644 index 000000000..146a09f09 --- /dev/null +++ b/doc/reference/internals/iproto/images/repl_join_meta.svg @@ -0,0 +1,28 @@ +IPROTO_JOIN_METASizeMP_UINTHeaderIPROTO_REQUEST_TYPEIPROTO_JOIN_METAIPROTO_SYNCMP_UINT \ No newline at end of file diff --git a/doc/reference/internals/iproto/images/repl_join_request.puml b/doc/reference/internals/iproto/images/repl_join_request.puml index 849f6ad47..b7841817e 100644 --- a/doc/reference/internals/iproto/images/repl_join_request.puml +++ b/doc/reference/internals/iproto/images/repl_join_request.puml @@ -14,7 +14,8 @@ json "**IPROTO_JOIN**" as join_request { "[[https://tarantool.io/en/doc/latest/reference/internals/iproto/keys IPROTO_SYNC]]": "MP_UINT" }, "Body": { - "[[https://tarantool.io/en/doc/latest/reference/internals/iproto/keys IPROTO_INSTANCE_UUID]]": "MP_STR – UUID of this instance" + "[[https://tarantool.io/en/doc/latest/reference/internals/iproto/keys IPROTO_INSTANCE_UUID]]": "MP_STR – UUID of this instance", + "[[https://tarantool.io/en/doc/latest/reference/internals/iproto/keys IPROTO_SERVER_VERSION]]": "MP_UINT – replica's version" } } diff --git a/doc/reference/internals/iproto/images/repl_join_request.svg b/doc/reference/internals/iproto/images/repl_join_request.svg index 96b5fb6f9..aec1fb634 100644 --- a/doc/reference/internals/iproto/images/repl_join_request.svg +++ b/doc/reference/internals/iproto/images/repl_join_request.svg @@ -1,4 +1,4 @@ -IPROTO_JOINSizeMP_UINTHeaderIPROTO_REQUEST_TYPEIPROTO_JOINIPROTO_SYNCMP_UINTBodyIPROTO_INSTANCE_UUIDMP_STR - UUID of this instance \ No newline at end of file diff --git a/doc/reference/internals/iproto/images/repl_register.puml b/doc/reference/internals/iproto/images/repl_register.puml new file mode 100644 index 000000000..8e08db1f3 --- /dev/null +++ b/doc/reference/internals/iproto/images/repl_register.puml @@ -0,0 +1,22 @@ +@startuml + +skinparam { + HyperlinkColor #0077FF + FontColor #313131 + BorderColor #313131 + BackgroundColor transparent +} + +json "**IPROTO_REGISTER**" as register_request { + "Size": "MP_UINT", + "Header": { + "[[https://tarantool.io/en/doc/latest/reference/internals/iproto/keys IPROTO_REQUEST_TYPE]]": "IPROTO_REGISTER", + "[[https://tarantool.io/en/doc/latest/reference/internals/iproto/keys IPROTO_SYNC]]": "MP_UINT" + }, + "Body": { + "[[https://tarantool.io/en/doc/latest/reference/internals/iproto/keys IPROTO_INSTANCE_UUID]]": "MP_STR – UUID of this instance", + "[[https://tarantool.io/en/doc/latest/reference/internals/iproto/keys IPROTO_VCLOCK]]": "MP_MAP" + } +} + +@enduml diff --git a/doc/reference/internals/iproto/images/repl_register.svg b/doc/reference/internals/iproto/images/repl_register.svg new file mode 100644 index 000000000..23325156d --- /dev/null +++ b/doc/reference/internals/iproto/images/repl_register.svg @@ -0,0 +1,32 @@ +IPROTO_REGISTERSizeMP_UINTHeaderIPROTO_REQUEST_TYPEIPROTO_REGISTERIPROTO_SYNCMP_UINTBodyIPROTO_INSTANCE_UUIDMP_STR - UUID of this instanceIPROTO_VCLOCKMP_MAP \ No newline at end of file diff --git a/doc/reference/internals/iproto/keys.rst b/doc/reference/internals/iproto/keys.rst index 4d7c9b421..162ed0cc4 100644 --- a/doc/reference/internals/iproto/keys.rst +++ b/doc/reference/internals/iproto/keys.rst @@ -641,7 +641,7 @@ Multiple operations make use of this key in different ways: :ref:`IPROTO_REPLACE `, :ref:`IPROTO_UPSERT ` - Tuple to be inserted - * - :ref:`IPROTO_UPSERT ` + * - :ref:`IPROTO_UPDATE ` - Operations to perform * - :ref:`IPROTO_AUTH ` - Array of 2 fields: diff --git a/doc/reference/internals/iproto/replication.rst b/doc/reference/internals/iproto/replication.rst index 1a86040e1..448d8cb64 100644 --- a/doc/reference/internals/iproto/replication.rst +++ b/doc/reference/internals/iproto/replication.rst @@ -42,20 +42,29 @@ General - 0x29 - Response to IPROTO_VOTE. Used during replica set bootstrap - * - IPROTO_FETCH_SNAPSHOT + * - :ref:`IPROTO_FETCH_SNAPSHOT ` - 0x45 - Fetch the master's snapshot and start anonymous replication. - See :ref:`replication.anon ` - * - IPROTO_REGISTER + * - :ref:`IPROTO_REGISTER ` - 0x46 - Register an anonymous replica so it is not anonymous anymore + + * - :ref:`IPROTO_JOIN_META ` + - 0x47 + - A request sent in response to IPROTO_JOIN or IPROTO_FETCH_SNAPSHOT + before the instance initialization information + + * - :ref:`IPROTO_JOIN_SNAPSHOT ` + - 0x48 + - A request sent in response to IPROTO_JOIN or IPROTO_FETCH_SNAPSHOT + after the instance initialization information The master also sends :ref:`heartbeat ` messages to the replicas. The heartbeat message's IPROTO_REQUEST_TYPE is ``0``. Below are details on individual replication requests. -For synchronous replication requests, see :ref:`below `. +For synchronous replication requests, see :ref:`internals-iproto-replication-synchronous`. .. _box_protocol-heartbeat: @@ -92,24 +101,45 @@ IPROTO_JOIN Code: 0x41. -To join a replica set, an instance must send an initial IPROTO_JOIN request to any node in the replica set: +To join a replica set, an instance must send an initial IPROTO_JOIN request to +the master instance of the replica set: .. raw:: html :file: images/repl_join_request.svg -The node that receives the request does the following in response: +.. iproto_fetch_snapshot_response_sequence_start + +The instance that receives the request sends the following messages in response: -#. It sends its vclock: +1. Its vclock: .. raw:: html :file: images/repl_join_response.svg -#. It sends a number of :ref:`INSERT ` requests (with additional LSN and ServerID). - In this way, the data is updated on the instance that sent the IPROTO_JOIN request. +2. (Optional) A sequence of requests with information required for instance initialization: + + - an :ref:`IPROTO_JOIN_META ` request + - an :ref:`IPROTO_RAFT ` request with IPROTO_RAFT_TERM and IPROTO_RAFT_VOTE fields + - an :ref:`IPROTO_RAFT_PROMOTE ` request + - an :ref:`IPROTO_JOIN_SNAPSHOT ` request + + This step applies if the IPROTO_SERVER_VERSION specified in the request is `2.10` or later. + +3. A number of :ref:`INSERT ` requests (with additional LSN and ServerID). + This way, the data is updated on the instance that sent the IPROTO_JOIN request. The instance should not reply to these INSERT requests. -#. It sends the new vclock's MP_MAP in a response similar to the one above - and closes the socket. +4. The new vclock's MP_MAP in a response similar to the one above. + +.. iproto_fetch_snapshot_response_sequence_end + +5. A number of :ref:`INSERT `, :ref:`REPLACE `, + :ref:`UPDATE `, :ref:`UPSERT `, + and :ref:`DELETE ` requests. This way, the instance + that is joining the replica set receives data updates that happened during + the join stage. + +6. The new vclock's MP_MAP in a response similar to the one above. .. _internals-iproto-replication-subscribe: @@ -135,6 +165,84 @@ is an optional key used in the SUBSCRIBE request followed by an array of ids of instances whose rows won't be relayed to the replica. The field is encoded only when the ID list is not empty. +.. _box_protocol-fetch-snapshot: + +IPROTO_FETCH_SNAPSHOT +~~~~~~~~~~~~~~~~~~~~~ + +Code: 0x45. + +To join a replica set as an anonymous replica, an instance must send an initial +IPROTO_FETCH_SNAPSHOT request to the master instance of the replica set: + +.. raw:: html + :file: images/repl_fetch_snapshot_request.svg + +To learn about anonymous replicas, see :ref:`replication.anon `. + +.. include:: replication.rst + :start-after: iproto_fetch_snapshot_response_sequence_start + :end-before: iproto_fetch_snapshot_response_sequence_end + +.. _box_protocol-register: + +IPROTO_REGISTER +~~~~~~~~~~~~~~~ + +Code: 0x46. + +To register an anonymous replica in a replica set so that it's not anonymous anymore, +it must send an IPROTO_REGISTER request to a master node of the replica set: + +.. raw:: html + :file: images/repl_register.svg + +The instance that receives the request sends the following messages in response: + +#. A number of :ref:`INSERT `, :ref:`REPLACE `, + :ref:`UPDATE `, :ref:`UPSERT `, + and :ref:`DELETE ` requests. This way, the instance + that is registering in the replica set receives data updates that happened + since the time it fetched the snapshot. + +#. The new vclock's MP_MAP. + +Technically, subsequent IPROTO_FETCH_SNAPSHOT and IPROTO_REGISTER requests are equivalent +to IPROTO_JOIN. + +.. _box_protocol-join-meta: + +IPROTO_JOIN_META +~~~~~~~~~~~~~~~~ + +Code: 0x47. + +When an instance receives an IPOTO_JOIN or IPROTO_FETCH_SNAPSHOT request, its responses +include the information required for the instance initialization: current Raft term, +current state of synchronous transaction queue. Before sending this information, +the instance sends an IPROTO_JOIN_META request with an empty body: + +.. raw:: html + :file: images/repl_join_meta.svg + +Learn more in :ref:`IPROTO_JOIN ` + +.. _box_protocol-join-snapshot: + +IPROTO_JOIN_SNAPSHOT +~~~~~~~~~~~~~~~~~~~~ + +Code: 0x48. + +An instance that has received an IPROTO_JOIN or IPROTO_FETCH_SNAPSHOT request +sends an IPROTO_JOIN_SNAPSHOT request with an empty body after it completes sending +the instance initialization information. + +.. raw:: html + :file: images/repl_join_snapshot.svg + +Learn more in :ref:`IPROTO_JOIN ` + .. _internals-iproto-replication-vote: IPROTO_VOTE @@ -187,7 +295,7 @@ Synchronous * - :ref:`IPROTO_RAFT ` - 0x1e - Inform that the node changed its :ref:`RAFT status ` - + * - :ref:`IPROTO_RAFT_PROMOTE ` - 0x1f - Wait, then choose new replication leader