Skip to content

Commit e37d4c6

Browse files
jan-ivarjan-ivar
committed
Bug 1868223 - Implement custom RTCEncodedAudioFrame::Read/WriteStructuredClone for same process. r=bwc
Differential Revision: https://phabricator.services.mozilla.com/D267544
1 parent 17b8e79 commit e37d4c6

4 files changed

Lines changed: 125 additions & 20 deletions

File tree

‎dom/base/StructuredCloneHolder.cpp‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,8 @@
5353
#include "mozilla/dom/MessagePortBinding.h"
5454
#include "mozilla/dom/OffscreenCanvas.h"
5555
#include "mozilla/dom/OffscreenCanvasBinding.h"
56+
#include "mozilla/dom/RTCEncodedAudioFrame.h"
57+
#include "mozilla/dom/RTCEncodedAudioFrameBinding.h"
5658
#include "mozilla/dom/RTCEncodedVideoFrame.h"
5759
#include "mozilla/dom/RTCEncodedVideoFrameBinding.h"
5860
#include "mozilla/dom/ReadableStream.h"
@@ -1169,6 +1171,17 @@ JSObject* StructuredCloneHolder::CustomReadHandler(
11691171
}
11701172
}
11711173

1174+
if (StaticPrefs::media_peerconnection_enabled() &&
1175+
aTag == SCTAG_DOM_RTCENCODEDAUDIOFRAME &&
1176+
CloneScope() == StructuredCloneScope::SameProcess &&
1177+
aCloneDataPolicy.areIntraClusterClonableSharedObjectsAllowed()) {
1178+
JS::Rooted<JSObject*> global(aCx, mGlobal->GetGlobalJSObject());
1179+
if (RTCEncodedAudioFrame_Binding::ConstructorEnabled(aCx, global)) {
1180+
return RTCEncodedAudioFrame::ReadStructuredClone(
1181+
aCx, mGlobal, aReader, RtcEncodedAudioFrames()[aIndex]);
1182+
}
1183+
}
1184+
11721185
return ReadFullySerializableObjects(aCx, aReader, aTag, false);
11731186
}
11741187

@@ -1322,6 +1335,17 @@ bool StructuredCloneHolder::CustomWriteHandler(
13221335
}
13231336
}
13241337

1338+
// See if this is an RTCEncodedAudioFrame object.
1339+
if (StaticPrefs::media_peerconnection_enabled()) {
1340+
RTCEncodedAudioFrame* rtcFrame = nullptr;
1341+
if (NS_SUCCEEDED(UNWRAP_OBJECT(RTCEncodedAudioFrame, &obj, rtcFrame))) {
1342+
SameProcessScopeRequired(aSameProcessScopeRequired);
1343+
return CloneScope() == StructuredCloneScope::SameProcess
1344+
? rtcFrame->WriteStructuredClone(aWriter, this)
1345+
: false;
1346+
}
1347+
}
1348+
13251349
{
13261350
// We only care about streams, so ReflectorToISupportsStatic is fine.
13271351
nsCOMPtr<nsISupports> base = xpc::ReflectorToISupportsStatic(aObj);

‎dom/base/StructuredCloneHolder.h‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,7 @@ class MessagePortIdentifier;
173173
struct VideoFrameSerializedData;
174174
struct AudioDataSerializedData;
175175
struct RTCEncodedVideoFrameData;
176+
struct RTCEncodedAudioFrameData;
176177

177178
class StructuredCloneHolder : public StructuredCloneHolderBase {
178179
public:
@@ -225,6 +226,7 @@ class StructuredCloneHolder : public StructuredCloneHolderBase {
225226
STMT(mEncodedVideoChunks); \
226227
STMT(mEncodedAudioChunks); \
227228
STMT(mRtcEncodedVideoFrames); \
229+
STMT(mRtcEncodedAudioFrames); \
228230
STMT(mPortIdentifiers);
229231

230232
// Call this method to know if this object is keeping some DOM object alive.
@@ -308,6 +310,10 @@ class StructuredCloneHolder : public StructuredCloneHolderBase {
308310
return mRtcEncodedVideoFrames;
309311
}
310312

313+
nsTArray<RTCEncodedAudioFrameData>& RtcEncodedAudioFrames() {
314+
return mRtcEncodedAudioFrames;
315+
}
316+
311317
// Implementations of the virtual methods to allow cloning of objects which
312318
// JS engine itself doesn't clone.
313319

@@ -425,6 +431,9 @@ class StructuredCloneHolder : public StructuredCloneHolderBase {
425431
// Used for cloning RTCEncodedVideoFrame in the structured cloning algorithm.
426432
nsTArray<RTCEncodedVideoFrameData> mRtcEncodedVideoFrames;
427433

434+
// Used for cloning RTCEncodedAudioFrame in the structured cloning algorithm.
435+
nsTArray<RTCEncodedAudioFrameData> mRtcEncodedAudioFrames;
436+
428437
// This raw pointer is only set within ::Read() and is unset by the end.
429438
nsIGlobalObject* MOZ_NON_OWNING_REF mGlobal;
430439

‎dom/media/webrtc/jsapi/RTCEncodedAudioFrame.cpp‎

Lines changed: 72 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
#include <memory>
1212
#include <utility>
1313

14+
#include "api/frame_transformer_factory.h"
1415
#include "api/frame_transformer_interface.h"
1516
#include "js/RootingAPI.h"
1617
#include "jsapi/RTCEncodedFrameBase.h"
@@ -20,6 +21,8 @@
2021
#include "mozilla/Unused.h"
2122
#include "mozilla/dom/RTCEncodedAudioFrameBinding.h"
2223
#include "mozilla/dom/RTCRtpScriptTransformer.h"
24+
#include "mozilla/dom/StructuredCloneHolder.h"
25+
#include "mozilla/dom/StructuredCloneTags.h"
2326
#include "mozilla/fallible.h"
2427
#include "nsContentUtils.h"
2528
#include "nsCycleCollectionParticipant.h"
@@ -29,8 +32,20 @@
2932

3033
namespace mozilla::dom {
3134

32-
NS_IMPL_CYCLE_COLLECTION_INHERITED(RTCEncodedAudioFrame, RTCEncodedFrameBase,
33-
mOwner)
35+
NS_IMPL_CYCLE_COLLECTION_CLASS(RTCEncodedAudioFrame)
36+
NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN_INHERITED(RTCEncodedAudioFrame,
37+
RTCEncodedFrameBase)
38+
NS_IMPL_CYCLE_COLLECTION_UNLINK(mOwner)
39+
NS_IMPL_CYCLE_COLLECTION_UNLINK_PRESERVED_WRAPPER
40+
NS_IMPL_CYCLE_COLLECTION_UNLINK_END
41+
NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN_INHERITED(RTCEncodedAudioFrame,
42+
RTCEncodedFrameBase)
43+
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mOwner)
44+
NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END
45+
NS_IMPL_CYCLE_COLLECTION_TRACE_BEGIN_INHERITED(RTCEncodedAudioFrame,
46+
RTCEncodedFrameBase)
47+
NS_IMPL_CYCLE_COLLECTION_TRACE_PRESERVED_WRAPPER
48+
NS_IMPL_CYCLE_COLLECTION_TRACE_END
3449
NS_IMPL_ADDREF_INHERITED(RTCEncodedAudioFrame, RTCEncodedFrameBase)
3550
NS_IMPL_RELEASE_INHERITED(RTCEncodedAudioFrame, RTCEncodedFrameBase)
3651

@@ -48,7 +63,7 @@ RTCEncodedAudioFrame::RTCEncodedAudioFrame(
4863
mMetadata.mSynchronizationSource.Construct(mFrame->GetSsrc());
4964
mMetadata.mPayloadType.Construct(mFrame->GetPayloadType());
5065
const auto& audioFrame(
51-
static_cast<webrtc::TransformableAudioFrameInterface&>(*mState.mFrame));
66+
static_cast<webrtc::TransformableAudioFrameInterface&>(*mFrame));
5267
mMetadata.mContributingSources.Construct();
5368
for (const auto csrc : audioFrame.GetContributingSources()) {
5469
Unused << mMetadata.mContributingSources.Value().AppendElement(csrc,
@@ -63,7 +78,22 @@ RTCEncodedAudioFrame::RTCEncodedAudioFrame(
6378
mozilla::HoldJSObjects(this);
6479
}
6580

81+
RTCEncodedAudioFrame::RTCEncodedAudioFrame(nsIGlobalObject* aGlobal,
82+
RTCEncodedAudioFrameData&& aData)
83+
: RTCEncodedAudioFrameData{{std::move(aData.mFrame), aData.mCounter,
84+
aData.mTimestamp},
85+
std::move(aData.mMetadata)},
86+
RTCEncodedFrameBase(aGlobal, static_cast<RTCEncodedFrameState&>(*this)),
87+
mOwner(nullptr) {
88+
// Base class needs this, but can't do it itself because of an assertion in
89+
// the cycle-collector.
90+
mozilla::HoldJSObjects(this);
91+
}
92+
6693
RTCEncodedAudioFrame::~RTCEncodedAudioFrame() {
94+
// Clear JS::Heap<> members before unregistering as a script holder,
95+
// so their destructors don't barrier against a finalized JS object.
96+
mData = nullptr; // from RTCEncodedFrameBase (protected)
6797
// Base class needs this, but can't do it itself because of an assertion in
6898
// the cycle-collector.
6999
mozilla::DropJSObjects(this);
@@ -74,6 +104,14 @@ JSObject* RTCEncodedAudioFrame::WrapObject(JSContext* aCx,
74104
return RTCEncodedAudioFrame_Binding::Wrap(aCx, this, aGivenProto);
75105
}
76106

107+
RTCEncodedAudioFrameData RTCEncodedAudioFrameData::Clone() const {
108+
return RTCEncodedAudioFrameData{
109+
{webrtc::CloneAudioFrame(
110+
static_cast<webrtc::TransformableAudioFrameInterface*>(
111+
mFrame.get()))},
112+
RTCEncodedAudioFrameMetadata(mMetadata)};
113+
}
114+
77115
nsIGlobalObject* RTCEncodedAudioFrame::GetParentObject() const {
78116
return mGlobal;
79117
}
@@ -89,17 +127,40 @@ bool RTCEncodedAudioFrame::CheckOwner(RTCRtpScriptTransformer* aOwner) const {
89127

90128
// https://www.w3.org/TR/webrtc-encoded-transform/#RTCEncodedAudioFrame-serialization
91129
/* static */
92-
already_AddRefed<RTCEncodedAudioFrame>
93-
RTCEncodedAudioFrame::ReadStructuredClone(JSContext* aCx,
94-
nsIGlobalObject* aGlobal,
95-
JSStructuredCloneReader* aReader) {
96-
// TBD implementation to follow in followup patch
97-
return nullptr;
130+
JSObject* RTCEncodedAudioFrame::ReadStructuredClone(
131+
JSContext* aCx, nsIGlobalObject* aGlobal, JSStructuredCloneReader* aReader,
132+
RTCEncodedAudioFrameData& aData) {
133+
JS::Rooted<JS::Value> value(aCx, JS::NullValue());
134+
// To avoid a rooting hazard error from returning a raw JSObject* before
135+
// running the RefPtr destructor, RefPtr needs to be destructed before
136+
// returning the raw JSObject*, which is why the RefPtr<RTCEncodedAudioFrame>
137+
// is created in the scope below. Otherwise, the static analysis infers the
138+
// RefPtr cannot be safely destructed while the unrooted return JSObject* is
139+
// on the stack.
140+
{
141+
auto frame = MakeRefPtr<RTCEncodedAudioFrame>(aGlobal, std::move(aData));
142+
if (!GetOrCreateDOMReflector(aCx, frame, &value) || !value.isObject()) {
143+
return nullptr;
144+
}
145+
}
146+
return value.toObjectOrNull();
98147
}
99148

100149
bool RTCEncodedAudioFrame::WriteStructuredClone(
101-
JSContext* aCx, JSStructuredCloneWriter* aWriter) const {
102-
return false;
150+
JSStructuredCloneWriter* aWriter, StructuredCloneHolder* aHolder) const {
151+
AssertIsOnOwningThread();
152+
153+
// Indexing the chunk and send the index to the receiver.
154+
const uint32_t index =
155+
static_cast<uint32_t>(aHolder->RtcEncodedAudioFrames().Length());
156+
// The serialization is limited to the same process scope so it's ok to
157+
// hand over a (copy of a) webrtc internal object here.
158+
//
159+
// TODO: optimize later once encoded source API materializes
160+
// .AppendElement(aHolder->IsTransferred(mData) ? Take() : Clone())
161+
aHolder->RtcEncodedAudioFrames().AppendElement(Clone());
162+
return !NS_WARN_IF(
163+
!JS_WriteUint32Pair(aWriter, SCTAG_DOM_RTCENCODEDAUDIOFRAME, index));
103164
}
104165

105166
} // namespace mozilla::dom

‎dom/media/webrtc/jsapi/RTCEncodedAudioFrame.h‎

Lines changed: 20 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -7,15 +7,19 @@
77
#ifndef MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCENCODEDAUDIOFRAME_H_
88
#define MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCENCODEDAUDIOFRAME_H_
99

10-
#include "jsapi/RTCEncodedFrameBase.h"
1110
#include "mozilla/RefPtr.h"
1211
#include "mozilla/dom/RTCEncodedAudioFrameBinding.h"
12+
#include "mozilla/dom/RTCEncodedFrameBase.h"
1313
#include "nsIGlobalObject.h"
1414

1515
namespace mozilla::dom {
1616

17+
class StructuredCloneHolder;
18+
1719
struct RTCEncodedAudioFrameData : RTCEncodedFrameState {
1820
RTCEncodedAudioFrameMetadata mMetadata;
21+
22+
[[nodiscard]] RTCEncodedAudioFrameData Clone() const;
1923
};
2024

2125
// Wraps a libwebrtc frame, allowing the frame buffer to be modified, and
@@ -30,10 +34,13 @@ class RTCEncodedAudioFrame final : public RTCEncodedAudioFrameData,
3034
std::unique_ptr<webrtc::TransformableFrameInterface> aFrame,
3135
uint64_t aCounter, RTCRtpScriptTransformer* aOwner);
3236

37+
explicit RTCEncodedAudioFrame(nsIGlobalObject* aGlobal,
38+
RTCEncodedAudioFrameData&& aData);
39+
3340
// nsISupports
3441
NS_DECL_ISUPPORTS_INHERITED
35-
NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(RTCEncodedAudioFrame,
36-
RTCEncodedFrameBase)
42+
NS_DECL_CYCLE_COLLECTION_SCRIPT_HOLDER_CLASS_INHERITED(RTCEncodedAudioFrame,
43+
RTCEncodedFrameBase)
3744

3845
// webidl (timestamp and data accessors live in base class)
3946
JSObject* WrapObject(JSContext* aCx,
@@ -47,12 +54,11 @@ class RTCEncodedAudioFrame final : public RTCEncodedAudioFrameData,
4754

4855
bool IsVideo() const override { return false; }
4956

50-
// [Serializable] implementations: {Read, Write}StructuredClone
51-
static already_AddRefed<RTCEncodedAudioFrame> ReadStructuredClone(
52-
JSContext* aCx, nsIGlobalObject* aGlobal,
53-
JSStructuredCloneReader* aReader);
54-
bool WriteStructuredClone(JSContext* aCx,
55-
JSStructuredCloneWriter* aWriter) const;
57+
static JSObject* ReadStructuredClone(JSContext* aCx, nsIGlobalObject* aGlobal,
58+
JSStructuredCloneReader* aReader,
59+
RTCEncodedAudioFrameData& aData);
60+
bool WriteStructuredClone(JSStructuredCloneWriter* aWriter,
61+
StructuredCloneHolder* aHolder) const;
5662

5763
private:
5864
virtual ~RTCEncodedAudioFrame();
@@ -63,6 +69,11 @@ class RTCEncodedAudioFrame final : public RTCEncodedAudioFrameData,
6369
RTCEncodedAudioFrame(RTCEncodedAudioFrame&&) = delete;
6470
RTCEncodedAudioFrame& operator=(RTCEncodedAudioFrame&&) = delete;
6571

72+
// RTCEncodedAudioFrame can run on either main thread or worker thread.
73+
void AssertIsOnOwningThread() const {
74+
NS_ASSERT_OWNINGTHREAD(RTCEncodedAudioFrame);
75+
}
76+
6677
RefPtr<RTCRtpScriptTransformer> mOwner;
6778
};
6879

0 commit comments

Comments
 (0)