Match header extensions to remote media sections

Firefox would send updated header extension
in renegotiation, e.g. publish a track without
simucalst then renegotiate second track with
simucalst, the two media secontions will have
different rtp header extensions in offer. Need
to match remote header extentions for each
media sections to avoid second track publish
failed.

Author: cnderrauber

mediaengine.go

@@ -456,6 +456,30 @@ func (m *MediaEngine) matchRemoteCodec(remoteCodec RTPCodecParameters, typ RTPCo
 	return matchType, nil
 }
 
+// Update header extensions from a remote media section
+func (m *MediaEngine) updateHeaderExtensionFromMediaSection(media *sdp.MediaDescription) error {
+	var typ RTPCodecType
+	switch {
+	case strings.EqualFold(media.MediaName.Media, "audio"):
+		typ = RTPCodecTypeAudio
+	case strings.EqualFold(media.MediaName.Media, "video"):
+		typ = RTPCodecTypeVideo
+	default:
+		return nil
+	}
+	extensions, err := rtpExtensionsFromMediaDescription(media)
+	if err != nil {
+		return err
+	}
+
+	for extension, id := range extensions {
+		if err = m.updateHeaderExtension(id, extension, typ); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // Look up a header extension and enable if it exists
 func (m *MediaEngine) updateHeaderExtension(id int, extension string, typ RTPCodecType) error {
 	if m.negotiatedHeaderExtensions == nil {
@@ -499,14 +523,27 @@ func (m *MediaEngine) updateFromRemoteDescription(desc sdp.SessionDescription) e
 
 	for _, media := range desc.MediaDescriptions {
 		var typ RTPCodecType
+
 		switch {
-		case !m.negotiatedAudio && strings.EqualFold(media.MediaName.Media, "audio"):
-			m.negotiatedAudio = true
+		case strings.EqualFold(media.MediaName.Media, "audio"):
 			typ = RTPCodecTypeAudio
-		case !m.negotiatedVideo && strings.EqualFold(media.MediaName.Media, "video"):
-			m.negotiatedVideo = true
+		case strings.EqualFold(media.MediaName.Media, "video"):
 			typ = RTPCodecTypeVideo
+		}
+
+		switch {
+		case !m.negotiatedAudio && typ == RTPCodecTypeAudio:
+			m.negotiatedAudio = true
+		case !m.negotiatedVideo && typ == RTPCodecTypeVideo:
+			m.negotiatedVideo = true
 		default:
+			// update header extesions from remote sdp if codec is negotiated, Firefox
+			// would send updated header extension in renegotiation.
+			// e.g. publish first track without simucalst ->negotiated-> publish second track with simucalst
+			// then the two media secontions have different rtp header extensions in offer
+			if err := m.updateHeaderExtensionFromMediaSection(media); err != nil {
+				return err
+			}
 			continue
 		}
 
@@ -542,16 +579,9 @@ func (m *MediaEngine) updateFromRemoteDescription(desc sdp.SessionDescription) e
 			continue
 		}
 
-		extensions, err := rtpExtensionsFromMediaDescription(media)
-		if err != nil {
+		if err := m.updateHeaderExtensionFromMediaSection(media); err != nil {
 			return err
 		}
-
-		for extension, id := range extensions {
-			if err = m.updateHeaderExtension(id, extension, typ); err != nil {
-				return err
-			}
-		}
 	}
 	return nil
 }

mediaengine_test.go

@@ -212,6 +212,39 @@ a=rtpmap:111 opus/48000/2
 		assert.False(t, midVideoEnabled)
 	})
 
+	t.Run("Different Header Extensions on same codec", func(t *testing.T) {
+		const headerExtensions = `v=0
+o=- 4596489990601351948 2 IN IP4 127.0.0.1
+s=-
+t=0 0
+m=audio 9 UDP/TLS/RTP/SAVPF 111
+a=rtpmap:111 opus/48000/2
+m=audio 9 UDP/TLS/RTP/SAVPF 111
+a=extmap:7 urn:ietf:params:rtp-hdrext:sdes:mid
+a=extmap:5 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id
+a=rtpmap:111 opus/48000/2
+`
+
+		m := MediaEngine{}
+		assert.NoError(t, m.RegisterDefaultCodecs())
+		assert.NoError(t, m.RegisterHeaderExtension(RTPHeaderExtensionCapability{URI: "urn:ietf:params:rtp-hdrext:sdes:mid"}, RTPCodecTypeAudio))
+		assert.NoError(t, m.RegisterHeaderExtension(RTPHeaderExtensionCapability{URI: "urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id"}, RTPCodecTypeAudio))
+		assert.NoError(t, m.updateFromRemoteDescription(mustParse(headerExtensions)))
+
+		assert.False(t, m.negotiatedVideo)
+		assert.True(t, m.negotiatedAudio)
+
+		absID, absAudioEnabled, absVideoEnabled := m.getHeaderExtensionID(RTPHeaderExtensionCapability{sdp.ABSSendTimeURI})
+		assert.Equal(t, absID, 0)
+		assert.False(t, absAudioEnabled)
+		assert.False(t, absVideoEnabled)
+
+		midID, midAudioEnabled, midVideoEnabled := m.getHeaderExtensionID(RTPHeaderExtensionCapability{sdp.SDESMidURI})
+		assert.Equal(t, midID, 7)
+		assert.True(t, midAudioEnabled)
+		assert.False(t, midVideoEnabled)
+	})
+
 	t.Run("Prefers exact codec matches", func(t *testing.T) {
 		const profileLevels = `v=0
 o=- 4596489990601351948 2 IN IP4 127.0.0.1

peerconnection.go

@@ -2445,7 +2445,9 @@ func (pc *PeerConnection) generateMatchedSDP(transceivers []*RTPTransceiver, use
 				sender.setNegotiated()
 			}
 			mediaTransceivers := []*RTPTransceiver{t}
-			mediaSections = append(mediaSections, mediaSection{id: midValue, transceivers: mediaTransceivers, ridMap: getRids(media)})
+
+			extensions, _ := rtpExtensionsFromMediaDescription(media)
+			mediaSections = append(mediaSections, mediaSection{id: midValue, transceivers: mediaTransceivers, matchExtensions: extensions, ridMap: getRids(media)})
 		}
 	}
 

peerconnection_media_test.go

@@ -1090,6 +1090,9 @@ func TestPeerConnection_Simulcast_Probe(t *testing.T) {
 			return
 		}))
 
+		peerConnectionConnected := untilConnectionState(PeerConnectionStateConnected, pcOffer, pcAnswer)
+		peerConnectionConnected.Wait()
+
 		sequenceNumber := uint16(0)
 		sendRTPPacket := func() {
 			sequenceNumber++
@@ -1107,13 +1110,13 @@ func TestPeerConnection_Simulcast_Probe(t *testing.T) {
 			sendRTPPacket()
 		}
 
-		assert.NoError(t, signalPair(pcOffer, pcAnswer))
-
 		trackRemoteChan := make(chan *TrackRemote, 1)
 		pcAnswer.OnTrack(func(trackRemote *TrackRemote, _ *RTPReceiver) {
 			trackRemoteChan <- trackRemote
 		})
 
+		assert.NoError(t, signalPair(pcOffer, pcAnswer))
+
 		trackRemote := func() *TrackRemote {
 			for {
 				select {

sdp.go

@@ -487,6 +487,11 @@ func addTransceiverSDP(
 
 	parameters := mediaEngine.getRTPParametersByKind(t.kind, directions)
 	for _, rtpExtension := range parameters.HeaderExtensions {
+		if mediaSection.matchExtensions != nil {
+			if _, enabled := mediaSection.matchExtensions[rtpExtension.URI]; !enabled {
+				continue
+			}
+		}
 		extURL, err := url.Parse(rtpExtension.URI)
 		if err != nil {
 			return false, err
@@ -533,10 +538,11 @@ type simulcastRid struct {
 }
 
 type mediaSection struct {
-	id           string
-	transceivers []*RTPTransceiver
-	data         bool
-	ridMap       map[string]*simulcastRid
+	id              string
+	transceivers    []*RTPTransceiver
+	data            bool
+	matchExtensions map[string]int
+	ridMap          map[string]*simulcastRid
 }
 
 func bundleMatchFromRemote(matchBundleGroup *string) func(mid string) bool {