diff --git a/internal/rtpbuffer/packet_factory.go b/internal/rtpbuffer/packet_factory.go
index d3f5a12d..8bb49823 100644
--- a/internal/rtpbuffer/packet_factory.go
+++ b/internal/rtpbuffer/packet_factory.go
@@ -94,8 +94,6 @@ func (m *PacketFactoryCopy) NewPacket(header *rtp.Header, payload []byte, rtxSsr
 		p.header.SSRC = rtxSsrc
 		// Rewrite the payload type.
 		p.header.PayloadType = rtxPayloadType
-		// Rewrite the sequence number.
-		p.header.SequenceNumber = m.rtxSequencer.NextSequenceNumber()
 		// Remove padding if present.
 		if p.header.Padding && p.payload != nil && len(p.payload) > 0 {
 			paddingLength := int(p.payload[len(p.payload)-1])
diff --git a/pkg/nack/responder_interceptor.go b/pkg/nack/responder_interceptor.go
index 58e34301..2a2117ac 100644
--- a/pkg/nack/responder_interceptor.go
+++ b/pkg/nack/responder_interceptor.go
@@ -25,6 +25,7 @@ func (r *ResponderInterceptorFactory) NewInterceptor(_ string) (interceptor.Inte
 		size:          1024,
 		log:           logging.NewDefaultLoggerFactory().NewLogger("nack_responder"),
 		streams:       map[uint32]*localStream{},
+		rtxSequencer:  rtp.NewRandomSequencer(),
 	}
 
 	for _, opt := range r.opts {
@@ -54,6 +55,8 @@ type ResponderInterceptor struct {
 
 	streams   map[uint32]*localStream
 	streamsMu sync.Mutex
+
+	rtxSequencer rtp.Sequencer
 }
 
 type localStream struct {
@@ -147,6 +150,7 @@ func (n *ResponderInterceptor) resendPackets(nack *rtcp.TransportLayerNack) {
 			defer stream.rtpBufferMutex.Unlock()
 
 			if p := stream.rtpBuffer.Get(seq); p != nil {
+				p.Header().SequenceNumber = n.rtxSequencer.NextSequenceNumber()
 				if _, err := stream.rtpWriter.Write(p.Header(), p.Payload(), interceptor.Attributes{}); err != nil {
 					n.log.Warnf("failed resending nacked packet: %+v", err)
 				}
diff --git a/pkg/nack/responder_interceptor_test.go b/pkg/nack/responder_interceptor_test.go
index 019d85e5..9ff6cde5 100644
--- a/pkg/nack/responder_interceptor_test.go
+++ b/pkg/nack/responder_interceptor_test.go
@@ -76,12 +76,16 @@ func TestResponderInterceptor(t *testing.T) {
 					},
 				},
 			})
-
+			expectedSequenceNumber := uint16(0)
 			// seq number 13 was never sent, so it can't be resent
-			for _, seqNum := range []uint16{11, 12, 15} {
+			for range []uint16{11, 12, 15} {
 				select {
 				case p := <-stream.WrittenRTP():
-					require.Equal(t, seqNum, p.SequenceNumber)
+					if expectedSequenceNumber == 0 {
+						expectedSequenceNumber = p.SequenceNumber
+					}
+					require.Equal(t, expectedSequenceNumber, p.SequenceNumber)
+					expectedSequenceNumber++
 				case <-time.After(10 * time.Millisecond):
 					t.Fatal("written rtp packet not found")
 				}