From a7290a20695db369b933dde89e650807b845e6db Mon Sep 17 00:00:00 2001 From: Alessandro Ros Date: Sat, 14 Mar 2026 16:59:05 +0100 Subject: [PATCH] webrtc: support publishing multiple video/audio renditions (#5573) this allows to receive multiple video tracks from OBS Studio with the new WebRTC Simulcast feature introduced in v32.1.0. --- .github/workflows/release.yml | 4 +- README.md | 6 +- docs/1-kickoff/1-introduction.md | 6 +- docs/1-kickoff/2-install.md | 21 ++- docs/1-kickoff/3-upgrade.md | 2 +- docs/2-publish/17-obs-studio.md | 20 ++- docs/4-other/23-rtsp-specific-features.md | 4 +- docs/6-misc/3-security.md | 4 +- internal/protocols/webrtc/incoming_track.go | 3 +- internal/protocols/webrtc/peer_connection.go | 22 ++- .../protocols/webrtc/peer_connection_test.go | 153 ++++++++++++++++++ internal/protocols/webrtc/to_stream.go | 22 +-- internal/protocols/webrtc/to_stream_test.go | 10 +- 13 files changed, 240 insertions(+), 37 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 77a9f314..f3d7d7b2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -61,9 +61,9 @@ jobs: + `\n` + `## Security\n` + `\n` - + `Binaries are compiled from source code by the [Release workflow](https://github.com/${owner}/${repo}/actions/workflows/release.yml) without human intervention.\n` + + `Binaries are compiled from source code by the [Release workflow](https://github.com/${owner}/${repo}/actions/workflows/release.yml), which is a fully-visible process that prevents any change or external interference in produced artifacts.\n` + `\n` - + 'You can verify that binaries have been produced by the workflow by using [GitHub Attestations](https://docs.github.com/en/actions/concepts/security/artifact-attestations):\n' + + 'Checksums of binaries are also published in a public blockchain by using [GitHub Attestations](https://docs.github.com/en/actions/concepts/security/artifact-attestations), and they can be verified by running:\n' + `\n` + '```\n' + `ls mediamtx_* | xargs -L1 gh attestation verify --repo bluenviron/mediamtx\n` diff --git a/README.md b/README.md index c12e5346..3cf4aba7 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@
-_MediaMTX_ is a ready-to-use and zero-dependency real-time media server and media proxy that allows to publish, read, proxy, record and playback video and audio streams. It has been conceived as a "media router" that routes media streams from one end to the other. +_MediaMTX_ is a ready-to-use and zero-dependency real-time media server and media proxy that allows to publish, read, proxy, record and playback video and audio streams. It has been conceived as a "media router" that routes media streams from one end to the other, with a focus on efficiency and portability.
@@ -27,8 +27,8 @@ _MediaMTX_ is a ready-to-use and zero-dependency real-time media server and medi

Features

-- [Publish](https://mediamtx.org/docs/usage/publish) live streams to the server with SRT, WebRTC, RTSP, RTMP, HLS, MPEG-TS, RTP -- [Read](https://mediamtx.org/docs/usage/read) live streams from the server with SRT, WebRTC, RTSP, RTMP, HLS +- [Publish](https://mediamtx.org/docs/usage/publish) live streams to the server with SRT, WebRTC, RTSP, RTMP, HLS, MPEG-TS, RTP, using FFmpeg, GStreamer, OBS Studio, Python , Golang, Unity, web browsers, Raspberry Pi Cameras and more. +- [Read](https://mediamtx.org/docs/usage/read) live streams from the server with SRT, WebRTC, RTSP, RTMP, HLS, using FFmpeg, GStreamer, VLC, OBS Studio, Python , Golang, Unity, web browsers and more. - Streams are automatically converted from a protocol to another - Serve several streams at once in separate paths - Reload the configuration without disconnecting existing clients (hot reloading) diff --git a/docs/1-kickoff/1-introduction.md b/docs/1-kickoff/1-introduction.md index 27506559..1504793d 100644 --- a/docs/1-kickoff/1-introduction.md +++ b/docs/1-kickoff/1-introduction.md @@ -2,12 +2,12 @@ Welcome to the MediaMTX documentation! -_MediaMTX_ is a ready-to-use and zero-dependency live media server and media proxy. It has been conceived as a "media router" that routes media streams from one end to the other. +_MediaMTX_ is a ready-to-use and zero-dependency live media server and media proxy. It has been conceived as a "media router" that routes media streams from one end to the other, with a focus on efficiency and portability. Main features: -- [Publish](../2-publish/01-overview.md) live streams to the server with SRT, WebRTC, RTSP, RTMP, HLS, MPEG-TS, RTP -- [Read](../3-read/01-overview.md) live streams from the server with SRT, WebRTC, RTSP, RTMP, HLS +- [Publish](../2-publish/01-overview.md) live streams to the server with SRT, WebRTC, RTSP, RTMP, HLS, MPEG-TS, RTP, using FFmpeg, GStreamer, OBS Studio, Python , Golang, Unity, Web browsers, Raspberry Pi Cameras and more. +- [Read](../3-read/01-overview.md) live streams from the server with SRT, WebRTC, RTSP, RTMP, HLS, using FFmpeg, GStreamer, VLC, OBS Studio, Python , Golang, Unity, Web browsers and more. - Streams are automatically converted from a protocol to another - Serve several streams at once in separate paths - Reload the configuration without disconnecting existing clients (hot reloading) diff --git a/docs/1-kickoff/2-install.md b/docs/1-kickoff/2-install.md index 752edb41..d6e90a66 100644 --- a/docs/1-kickoff/2-install.md +++ b/docs/1-kickoff/2-install.md @@ -1,6 +1,12 @@ # Install -There are several installation methods available: standalone binary, Docker image, Arch Linux package, FreeBSD or OpenWrt binary. +There are several installation methods available: + +- [Standalone binary](#standalone-binary): use this if you are running Windows, macOS or you just want to try out _MediaMTX_. +- [Docker image](#docker-image): use this if you want to run _MediaMTX_ in an isolated and deterministic way. This is recommended for production environments. +- [Arch Linux package](#arch-linux-package): use this if you are running Arch Linux. +- [FreeBSD package](#freebsd-package): use this if you are running FreeBSD. +- [OpenWrt binary](#openwrt-binary): use this if you are running OpenWrt. ## Standalone binary @@ -48,27 +54,30 @@ There are four image variants: The `1` tag corresponds to the latest `1.x.x` release, that should guarantee backward compatibility when upgrading. It is also possible to bind the image to a specific release, by using the release name as tag (`bluenviron/mediamtx:{docker_version_tag}`). -The base image does not contain any utility, in order to minimize size and frequency of updates. If you need additional software (like curl, wget, GStreamer), you can build a custom image by using the _MediaMTX_ image as a base stage, by creating a file named `Dockerfile` with this content: +The base image does not contain any utility, in order to minimize size and frequency of updates. If you need additional software (like curl, wget, GStreamer), you can build a custom image by creating a file named `Dockerfile` with this content: -``` +```Dockerfile FROM bluenviron/mediamtx:1 AS mediamtx FROM ubuntu:24.04 COPY --from=mediamtx /mediamtx / COPY --from=mediamtx /mediamtx.yml / +# add anything you need. RUN apt update && apt install -y \ - (insert additional utilities here) + gstreamer1.0-tools ENTRYPOINT [ "/mediamtx" ] ``` And then build it: -``` +```sh docker build . -t my-mediamtx ``` +In particular, the custom image is using the official _MediaMTX_ image as a base stage, and then adds a Linux-based operating system on top of it. Since _MediaMTX_ binaries are not tied to a specific Linux distribution or version, you can use anything you like. + ## Arch Linux package If you are running the Arch Linux distribution, launch: @@ -79,7 +88,7 @@ cd mediamtx makepkg -si ``` -## FreeBSD +## FreeBSD package Available via ports tree or using packages (2025Q2 and later) as listed below: diff --git a/docs/1-kickoff/3-upgrade.md b/docs/1-kickoff/3-upgrade.md index bdd12be5..c6fdaf25 100644 --- a/docs/1-kickoff/3-upgrade.md +++ b/docs/1-kickoff/3-upgrade.md @@ -33,7 +33,7 @@ Then recreate the container as described in [Install](2-install.md#docker-image) Repeat the installation procedure. -## FreeBSD +## FreeBSD package Repeat the installation procedure. diff --git a/docs/2-publish/17-obs-studio.md b/docs/2-publish/17-obs-studio.md index f485adae..85b963c3 100644 --- a/docs/2-publish/17-obs-studio.md +++ b/docs/2-publish/17-obs-studio.md @@ -29,7 +29,7 @@ Then use the button `Start Recording` (instead of `Start Streaming`) to start st ## OBS Studio and RTMP, multitrack video -OBS Studio can publish multiple video tracks or renditions at once. Make sure that the OBS Studio version is ≥ 31.0.0. Open `Settings -> Stream` and use the following parameters: +OBS Studio can publish multiple video tracks or renditions at once (simulcast). Make sure that the OBS Studio version is ≥ 31.0.0. Open `Settings -> Stream` and use the following parameters: - Service: `Custom...` - Server: `rtmp://localhost/mystream` @@ -105,6 +105,10 @@ OBS Studio can publish multiple video tracks or renditions at once. Make sure th - `obs_qsv11_v2`: QuickSync H264 - `obs_x264`: software H264 +Save the configuration and click `Start streaming`. + +The resulting stream will be available on path `/mystream`. + ## OBS Studio and WebRTC Recent versions of OBS Studio can also publish streams to the server with the [WebRTC / WHIP protocol](04-webrtc-clients.md) Use the following parameters: @@ -115,3 +119,17 @@ Recent versions of OBS Studio can also publish streams to the server with the [W Save the configuration and click `Start streaming`. The resulting stream will be available on path `/mystream`. + +## OBS Studio and WebRTC, multitrack video + +OBS Studio can publish multiple video tracks or renditions at once (simulcast) with WebRTC / WHIP too. Make sure that the OBS Studio version is ≥ 32.1.0. Open `Settings -> Stream` and use the following parameters: + +- Service: `WHIP` +- Server: `http://localhost:8889/mystream/whip` +- Simulcast, Total Layers: `2` (or greater) + +Currently it's not possible to change resolution or bitrate (or canvas) of renditions, since quality of secondary renditions is hardcoded as a percentage of the main one. You can find details on the [OBS documentation](https://obsproject.com/kb/whip-streaming-guide). + +Save the configuration and click `Start streaming`. + +The resulting stream will be available on path `/mystream`. diff --git a/docs/4-other/23-rtsp-specific-features.md b/docs/4-other/23-rtsp-specific-features.md index 3b666b6c..73ed973f 100644 --- a/docs/4-other/23-rtsp-specific-features.md +++ b/docs/4-other/23-rtsp-specific-features.md @@ -94,9 +94,9 @@ In environments where HTTP is the only protocol available for exposing services - RTSP over WebSocket: more efficient, requires WebSocket support from the gateway / firewall - RTSP over HTTP: older variant, should work even in extreme cases -_MediaMTX_ is automatically able to handle incoming HTTP tunneled connections, without any configuration required. +_MediaMTX_ is automatically able to handle incoming HTTP-tunneled RTSP connections without any configuration required. -In order to read a RTSP from an external server using HTTP tunneling, you can use the `rtsp+http` scheme: +In order to read a stream from an external RTSP server using HTTP tunneling, you can use the `rtsp+http` scheme: ```yml paths: diff --git a/docs/6-misc/3-security.md b/docs/6-misc/3-security.md index 0da3a21b..a9bf580f 100644 --- a/docs/6-misc/3-security.md +++ b/docs/6-misc/3-security.md @@ -2,13 +2,13 @@ ## Security of released binaries -Binaries published in the [Releases](https://github.com/bluenviron/mediamtx/releases) section of GitHub are the output of a process that is fully visible, both in terms of "ingredients" (the source code) and "recipe" (the process steps), and verifiable by third parties. This should prevent external interferences and guarantee security. This is the process: +Binaries published in the [Releases](https://github.com/bluenviron/mediamtx/releases) section of GitHub are the output of a building process that is fully visible, prevents hidden changes or external interferences in published artifacts, and allows validation by third parties: 1. During every release, the [Release workflow](https://github.com/bluenviron/mediamtx/actions/workflows/release.yml) is triggered on GitHub. 2. The release workflow pulls the source code and builds binaries. -3. The release workflow computes SHA256 checksums of binaries and publishes them to the Sigstore Public Good Instance through [GitHub Attestations](https://docs.github.com/en/actions/concepts/security/artifact-attestations). +3. The release workflow computes SHA256 checksums of binaries and publishes them to a public blockchain (Sigstore Public Good Instance) through [GitHub Attestations](https://docs.github.com/en/actions/concepts/security/artifact-attestations). 4. Checksums and binaries are published on the Release page. diff --git a/internal/protocols/webrtc/incoming_track.go b/internal/protocols/webrtc/incoming_track.go index 88bb2abf..0cd1345a 100644 --- a/internal/protocols/webrtc/incoming_track.go +++ b/internal/protocols/webrtc/incoming_track.go @@ -240,6 +240,7 @@ type IncomingTrack struct { track *webrtc.TrackRemote receiver *webrtc.RTPReceiver + rid string writeRTCP func([]rtcp.Packet) error log logger.Writer @@ -299,7 +300,7 @@ func (t *IncomingTrack) start() { go func() { buf := make([]byte, 1500) for { - n, _, err2 := t.receiver.Read(buf) + n, _, err2 := t.receiver.ReadSimulcast(buf, t.rid) if err2 != nil { return } diff --git a/internal/protocols/webrtc/peer_connection.go b/internal/protocols/webrtc/peer_connection.go index e62c012c..377924bd 100644 --- a/internal/protocols/webrtc/peer_connection.go +++ b/internal/protocols/webrtc/peer_connection.go @@ -59,6 +59,25 @@ func interfaceIPs(interfaceList []string) ([]string, error) { return ips, nil } +func maxTrackCount(medias []*sdp.MediaDescription) int { + total := 0 + for _, media := range medias { + ridCount := 0 + + for _, attr := range media.Attributes { + if attr.Key == "rid" { + ridCount++ + } + } + + if ridCount == 0 { + ridCount = 1 + } + total += ridCount + } + return total +} + // * skip ConfigureRTCPReports // * add statsInterceptor func registerInterceptors( @@ -696,7 +715,7 @@ func (co *PeerConnection) GatherIncomingTracks(timeout time.Duration) error { var sdp sdp.SessionDescription sdp.Unmarshal([]byte(co.wr.RemoteDescription().SDP)) //nolint:errcheck - maxTrackCount := len(sdp.MediaDescriptions) + maxTrackCount := maxTrackCount(sdp.MediaDescriptions) t := time.NewTimer(timeout) defer t.Stop() @@ -713,6 +732,7 @@ func (co *PeerConnection) GatherIncomingTracks(timeout time.Duration) error { t := &IncomingTrack{ track: pair.track, receiver: pair.receiver, + rid: pair.track.RID(), writeRTCP: co.wr.WriteRTCP, log: co.Log, } diff --git a/internal/protocols/webrtc/peer_connection_test.go b/internal/protocols/webrtc/peer_connection_test.go index 20ba17a4..59a47dd1 100644 --- a/internal/protocols/webrtc/peer_connection_test.go +++ b/internal/protocols/webrtc/peer_connection_test.go @@ -424,6 +424,159 @@ func TestPeerConnectionRead(t *testing.T) { }, pkts) } +func TestPeerConnectionReadSimulcast(t *testing.T) { + pub, err := webrtc.NewPeerConnection(webrtc.Configuration{}) + require.NoError(t, err) + defer pub.Close() //nolint:errcheck + + videoTrackL, err := webrtc.NewTrackLocalStaticRTP( + webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeVP8, + ClockRate: 90000, + }, + "video", "publisher", + webrtc.WithRTPStreamID("l"), + ) + require.NoError(t, err) + + videoTrackM, err := webrtc.NewTrackLocalStaticRTP( + webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeVP8, + ClockRate: 90000, + }, + "video", "publisher", + webrtc.WithRTPStreamID("m"), + ) + require.NoError(t, err) + + videoTrackH, err := webrtc.NewTrackLocalStaticRTP( + webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeVP8, + ClockRate: 90000, + }, + "video", "publisher", + webrtc.WithRTPStreamID("h"), + ) + require.NoError(t, err) + + transceiver, err := pub.AddTransceiverFromTrack(videoTrackL, webrtc.RTPTransceiverInit{ + Direction: webrtc.RTPTransceiverDirectionSendonly, + SendEncodings: []webrtc.RTPEncodingParameters{ + {RTPCodingParameters: webrtc.RTPCodingParameters{RID: "l"}}, + {RTPCodingParameters: webrtc.RTPCodingParameters{RID: "m"}}, + {RTPCodingParameters: webrtc.RTPCodingParameters{RID: "h"}}, + }, + }) + require.NoError(t, err) + + err = transceiver.Sender().AddEncoding(videoTrackM) + require.NoError(t, err) + + err = transceiver.Sender().AddEncoding(videoTrackH) + require.NoError(t, err) + + reader := &PeerConnection{ + LocalRandomUDP: true, + IPsFromInterfaces: true, + Publish: false, + Log: test.NilLogger, + } + err = reader.Start() + require.NoError(t, err) + defer reader.Close() + + offer, err := pub.CreateOffer(nil) + require.NoError(t, err) + + err = pub.SetLocalDescription(offer) + require.NoError(t, err) + + answer, err := reader.CreateFullAnswer(&offer) + require.NoError(t, err) + + err = pub.SetRemoteDescription(*answer) + require.NoError(t, err) + + err = reader.WaitUntilConnected(10 * time.Second) + require.NoError(t, err) + + var midExtID, ridExtID uint8 + for _, ext := range transceiver.Sender().GetParameters().HeaderExtensions { + switch ext.URI { + case sdp.SDESMidURI: + midExtID = uint8(ext.ID) + case sdp.SDESRTPStreamIDURI: + ridExtID = uint8(ext.ID) + } + } + require.NotZero(t, midExtID) + require.NotZero(t, ridExtID) + + mid := transceiver.Mid() + + go func() { + time.Sleep(200 * time.Millisecond) + + layers := []struct { + track *webrtc.TrackLocalStaticRTP + rid string + ssrc uint32 + seqNum uint16 + }{ + {videoTrackL, "l", 100001, 1000}, + {videoTrackM, "m", 100002, 2000}, + {videoTrackH, "h", 100003, 3000}, + } + + for i, layer := range layers { + pkt := &rtp.Packet{ + Header: rtp.Header{ + Version: 2, + Marker: true, + PayloadType: 96, + SequenceNumber: layer.seqNum, + Timestamp: 45343, + SSRC: layer.ssrc, + }, + Payload: []byte{5, 2}, + } + + pkt.ExtensionProfile = 0xBEDE + require.NoError(t, pkt.SetExtension(midExtID, []byte(mid))) + require.NoError(t, pkt.SetExtension(ridExtID, []byte(layer.rid))) + + err2 := layer.track.WriteRTP(pkt) + if err2 != nil { + return + } + layers[i].seqNum++ + } + }() + + err = reader.GatherIncomingTracks(5 * time.Second) + require.NoError(t, err) + + tracks := reader.IncomingTracks() + codecs := gatherCodecs(tracks) + + require.Equal(t, 3, len(codecs)) + + for _, codec := range codecs { + require.Equal(t, webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeVP8, + ClockRate: 90000, + RTCPFeedback: codec.RTCPFeedback, + }, codec.RTPCodecCapability) + } + + rids := make([]string, len(tracks)) + for i, track := range tracks { + rids[i] = track.track.RID() + } + sort.Strings(rids) + require.Equal(t, []string{"h", "l", "m"}, rids) +} + func TestPeerConnectionPublishRead(t *testing.T) { pc1 := &PeerConnection{ LocalRandomUDP: true, diff --git a/internal/protocols/webrtc/to_stream.go b/internal/protocols/webrtc/to_stream.go index 850a62d3..0cfb7b07 100644 --- a/internal/protocols/webrtc/to_stream.go +++ b/internal/protocols/webrtc/to_stream.go @@ -48,45 +48,45 @@ func ToStream( case strings.ToLower(webrtc.MimeTypeAV1): typ = description.MediaTypeVideo forma = &format.AV1{ - PayloadTyp: uint8(track.track.PayloadType()), + PayloadTyp: 96, } case strings.ToLower(webrtc.MimeTypeVP9): typ = description.MediaTypeVideo forma = &format.VP9{ - PayloadTyp: uint8(track.track.PayloadType()), + PayloadTyp: 96, } case strings.ToLower(webrtc.MimeTypeVP8): typ = description.MediaTypeVideo forma = &format.VP8{ - PayloadTyp: uint8(track.track.PayloadType()), + PayloadTyp: 96, } case strings.ToLower(webrtc.MimeTypeH265): typ = description.MediaTypeVideo forma = &format.H265{ - PayloadTyp: uint8(track.track.PayloadType()), + PayloadTyp: 96, } case strings.ToLower(webrtc.MimeTypeH264): typ = description.MediaTypeVideo forma = &format.H264{ - PayloadTyp: uint8(track.track.PayloadType()), + PayloadTyp: 96, PacketizationMode: 1, } case strings.ToLower(mimeTypeMultiopus): typ = description.MediaTypeAudio forma = &format.Opus{ - PayloadTyp: uint8(track.track.PayloadType()), + PayloadTyp: 96, ChannelCount: int(track.track.Codec().Channels), } case strings.ToLower(webrtc.MimeTypeOpus): typ = description.MediaTypeAudio forma = &format.Opus{ - PayloadTyp: uint8(track.track.PayloadType()), + PayloadTyp: 96, ChannelCount: func() int { if strings.Contains(track.track.Codec().SDPFmtpLine, "stereo=1") { return 2 @@ -109,7 +109,7 @@ func ToStream( forma = &format.G711{ PayloadTyp: func() uint8 { if channels > 1 { - return 118 + return 96 } return 0 }(), @@ -128,7 +128,7 @@ func ToStream( forma = &format.G711{ PayloadTyp: func() uint8 { if channels > 1 { - return 119 + return 96 } return 8 }(), @@ -140,7 +140,7 @@ func ToStream( case strings.ToLower(mimeTypeL16): typ = description.MediaTypeAudio forma = &format.LPCM{ - PayloadTyp: uint8(track.track.PayloadType()), + PayloadTyp: 96, BitDepth: 16, SampleRate: int(track.track.Codec().ClockRate), ChannelCount: int(track.track.Codec().Channels), @@ -187,6 +187,8 @@ func ToStream( } track.OnPacketRTP = func(pkt *rtp.Packet) { + pkt.PayloadType = forma.PayloadType() + pts, ok := timeDecoder.Decode(track, pkt) if !ok { return diff --git a/internal/protocols/webrtc/to_stream_test.go b/internal/protocols/webrtc/to_stream_test.go index b38e9cc2..5655f042 100644 --- a/internal/protocols/webrtc/to_stream_test.go +++ b/internal/protocols/webrtc/to_stream_test.go @@ -99,7 +99,7 @@ var toFromStreamCases = []struct { { "opus multichannel", &format.Opus{ - PayloadTyp: 112, + PayloadTyp: 96, ChannelCount: 6, }, webrtc.RTPCodecCapability{ @@ -116,7 +116,7 @@ var toFromStreamCases = []struct { { "opus stereo", &format.Opus{ - PayloadTyp: 111, + PayloadTyp: 96, ChannelCount: 2, }, webrtc.RTPCodecCapability{ @@ -133,7 +133,7 @@ var toFromStreamCases = []struct { { "opus mono", &format.Opus{ - PayloadTyp: 111, + PayloadTyp: 96, ChannelCount: 1, }, webrtc.RTPCodecCapability{ @@ -205,7 +205,7 @@ var toFromStreamCases = []struct { Channels: 2, }, &format.G711{ - PayloadTyp: 119, + PayloadTyp: 96, SampleRate: 8000, ChannelCount: 2, }, @@ -225,7 +225,7 @@ var toFromStreamCases = []struct { }, &format.G711{ MULaw: true, - PayloadTyp: 118, + PayloadTyp: 96, SampleRate: 8000, ChannelCount: 2, },