Bug 1868223 - Implement custom RTCEncodedVideoFrame::Read/WriteStructuredClone for same process. r=bwc,webidl,smaug

Differential Revision: https://phabricator.services.mozilla.com/D267543

Author: jan-ivar

dom/base/StructuredCloneHolder.cpp

@@ -22,6 +22,9 @@
 #include "mozilla/RefPtr.h"
 #include "mozilla/ScopeExit.h"
 #include "mozilla/StaticPrefs_dom.h"
+#ifdef MOZ_WEBRTC
+#  include "mozilla/StaticPrefs_media.h"
+#endif
 #include "mozilla/dom/AudioData.h"
 #include "mozilla/dom/BindingDeclarations.h"
 #include "mozilla/dom/BindingUtils.h"
@@ -52,6 +55,10 @@
 #include "mozilla/dom/MessagePortBinding.h"
 #include "mozilla/dom/OffscreenCanvas.h"
 #include "mozilla/dom/OffscreenCanvasBinding.h"
+#ifdef MOZ_WEBRTC
+#  include "mozilla/dom/RTCEncodedVideoFrame.h"
+#  include "mozilla/dom/RTCEncodedVideoFrameBinding.h"
+#endif
 #include "mozilla/dom/ReadableStream.h"
 #include "mozilla/dom/ReadableStreamBinding.h"
 #include "mozilla/dom/ScriptSettings.h"
@@ -1155,6 +1162,18 @@ JSObject* StructuredCloneHolder::CustomReadHandler(
     }
   }
 
+#ifdef MOZ_WEBRTC
+  if (StaticPrefs::media_peerconnection_enabled() &&
+      aTag == SCTAG_DOM_RTCENCODEDVIDEOFRAME &&
+      CloneScope() == StructuredCloneScope::SameProcess &&
+      aCloneDataPolicy.areIntraClusterClonableSharedObjectsAllowed()) {
+    JS::Rooted<JSObject*> global(aCx, mGlobal->GetGlobalJSObject());
+    if (RTCEncodedVideoFrame_Binding::ConstructorEnabled(aCx, global)) {
+      return RTCEncodedVideoFrame::ReadStructuredClone(
+          aCx, mGlobal, aReader, RtcEncodedVideoFrames()[aIndex]);
+    }
+  }
+#endif
   return ReadFullySerializableObjects(aCx, aReader, aTag, false);
 }
 
@@ -1297,6 +1316,18 @@ bool StructuredCloneHolder::CustomWriteHandler(
     }
   }
 
+#ifdef MOZ_WEBRTC
+  // See if this is an RTCEncodedVideoFrame object.
+  if (StaticPrefs::media_peerconnection_enabled()) {
+    RTCEncodedVideoFrame* rtcFrame = nullptr;
+    if (NS_SUCCEEDED(UNWRAP_OBJECT(RTCEncodedVideoFrame, &obj, rtcFrame))) {
+      SameProcessScopeRequired(aSameProcessScopeRequired);
+      return CloneScope() == StructuredCloneScope::SameProcess
+                 ? rtcFrame->WriteStructuredClone(aWriter, this)
+                 : false;
+    }
+  }
+#endif
   {
     // We only care about streams, so ReflectorToISupportsStatic is fine.
     nsCOMPtr<nsISupports> base = xpc::ReflectorToISupportsStatic(aObj);

dom/base/StructuredCloneHolder.h

@@ -172,6 +172,9 @@ class MessagePort;
 class MessagePortIdentifier;
 struct VideoFrameSerializedData;
 struct AudioDataSerializedData;
+#ifdef MOZ_WEBRTC
+struct RTCEncodedVideoFrameData;
+#endif
 
 class StructuredCloneHolder : public StructuredCloneHolderBase {
  public:
@@ -211,18 +214,25 @@ class StructuredCloneHolder : public StructuredCloneHolderBase {
             JS::MutableHandle<JS::Value> aValue,
             const JS::CloneDataPolicy& aCloneDataPolicy, ErrorResult& aRv);
 
-  // Create a statement for each of the side DOM-ish data members.
-  // mTransferredPorts is not included because it is part of the
-  // deserialized state.
-#define CLONED_DATA_MEMBERS  \
-  STMT(mBlobImplArray);      \
-  STMT(mWasmModuleArray);    \
-  STMT(mInputStreamArray);   \
-  STMT(mClonedSurfaces);     \
-  STMT(mVideoFrames);        \
-  STMT(mAudioData);          \
-  STMT(mEncodedVideoChunks); \
-  STMT(mEncodedAudioChunks); \
+#ifdef MOZ_WEBRTC
+#  define IF_WEBRTC(x) x
+#else
+#  define IF_WEBRTC(x)
+#endif
+
+// Create a statement for each of the side DOM-ish data members.
+// mTransferredPorts is not included because it is part of the
+// deserialized state.
+#define CLONED_DATA_MEMBERS                \
+  STMT(mBlobImplArray);                    \
+  STMT(mWasmModuleArray);                  \
+  STMT(mInputStreamArray);                 \
+  STMT(mClonedSurfaces);                   \
+  STMT(mVideoFrames);                      \
+  STMT(mAudioData);                        \
+  STMT(mEncodedVideoChunks);               \
+  STMT(mEncodedAudioChunks);               \
+  IF_WEBRTC(STMT(mRtcEncodedVideoFrames);) \
   STMT(mPortIdentifiers);
 
   // Call this method to know if this object is keeping some DOM object alive.
@@ -302,6 +312,12 @@ class StructuredCloneHolder : public StructuredCloneHolderBase {
     return mEncodedAudioChunks;
   }
 
+#ifdef MOZ_WEBRTC
+  nsTArray<RTCEncodedVideoFrameData>& RtcEncodedVideoFrames() {
+    return mRtcEncodedVideoFrames;
+  }
+#endif
+
   // Implementations of the virtual methods to allow cloning of objects which
   // JS engine itself doesn't clone.
 
@@ -416,6 +432,11 @@ class StructuredCloneHolder : public StructuredCloneHolderBase {
   // Used for cloning EncodedAudioChunk in the structured cloning algorithm.
   nsTArray<EncodedAudioChunkData> mEncodedAudioChunks;
 
+#ifdef MOZ_WEBRTC
+  // Used for cloning RTCEncodedVideoFrame in the structured cloning algorithm.
+  nsTArray<RTCEncodedVideoFrameData> mRtcEncodedVideoFrames;
+#endif
+
   // This raw pointer is only set within ::Read() and is unset by the end.
   nsIGlobalObject* MOZ_NON_OWNING_REF mGlobal;
 

dom/base/StructuredCloneTags.h

@@ -163,6 +163,10 @@ enum StructuredCloneTags : uint32_t {
 
   SCTAG_DOM_RTCDATACHANNEL,
 
+  SCTAG_DOM_RTCENCODEDVIDEOFRAME,
+
+  SCTAG_DOM_RTCENCODEDAUDIOFRAME,
+
   // IMPORTANT: If you plan to add an new IDB tag, it _must_ be add before the
   // "less stable" tags!
 };

dom/media/webrtc/jsapi/RTCEncodedAudioFrame.cpp

@@ -42,12 +42,13 @@ RTCEncodedAudioFrame::RTCEncodedAudioFrame(
     nsIGlobalObject* aGlobal,
     std::unique_ptr<webrtc::TransformableFrameInterface> aFrame,
     uint64_t aCounter, RTCRtpScriptTransformer* aOwner)
-    : RTCEncodedFrameBase(aGlobal, std::move(aFrame), aCounter),
+    : RTCEncodedAudioFrameData{{std::move(aFrame), aCounter, /*timestamp*/ 0}},
+      RTCEncodedFrameBase(aGlobal, static_cast<RTCEncodedFrameState&>(*this)),
       mOwner(aOwner) {
   mMetadata.mSynchronizationSource.Construct(mFrame->GetSsrc());
   mMetadata.mPayloadType.Construct(mFrame->GetPayloadType());
   const auto& audioFrame(
-      static_cast<webrtc::TransformableAudioFrameInterface&>(*mFrame));
+      static_cast<webrtc::TransformableAudioFrameInterface&>(*mState.mFrame));
   mMetadata.mContributingSources.Construct();
   for (const auto csrc : audioFrame.GetContributingSources()) {
     Unused << mMetadata.mContributingSources.Value().AppendElement(csrc,
@@ -85,4 +86,20 @@ void RTCEncodedAudioFrame::GetMetadata(
 bool RTCEncodedAudioFrame::CheckOwner(RTCRtpScriptTransformer* aOwner) const {
   return aOwner == mOwner;
 }
+
+// https://www.w3.org/TR/webrtc-encoded-transform/#RTCEncodedAudioFrame-serialization
+/* static */
+already_AddRefed<RTCEncodedAudioFrame>
+RTCEncodedAudioFrame::ReadStructuredClone(JSContext* aCx,
+                                          nsIGlobalObject* aGlobal,
+                                          JSStructuredCloneReader* aReader) {
+  // TBD implementation to follow in followup patch
+  return nullptr;
+}
+
+bool RTCEncodedAudioFrame::WriteStructuredClone(
+    JSContext* aCx, JSStructuredCloneWriter* aWriter) const {
+  return false;
+}
+
 }  // namespace mozilla::dom

dom/media/webrtc/jsapi/RTCEncodedAudioFrame.h

@@ -14,11 +14,16 @@
 
 namespace mozilla::dom {
 
+struct RTCEncodedAudioFrameData : RTCEncodedFrameState {
+  RTCEncodedAudioFrameMetadata mMetadata;
+};
+
 // Wraps a libwebrtc frame, allowing the frame buffer to be modified, and
 // providing read-only access to various metadata. After the libwebrtc frame is
 // extracted (with RTCEncodedFrameBase::TakeFrame), the frame buffer is
 // detached, but the metadata remains accessible.
-class RTCEncodedAudioFrame final : public RTCEncodedFrameBase {
+class RTCEncodedAudioFrame final : public RTCEncodedAudioFrameData,
+                                   public RTCEncodedFrameBase {
  public:
   explicit RTCEncodedAudioFrame(
       nsIGlobalObject* aGlobal,
@@ -42,10 +47,23 @@ class RTCEncodedAudioFrame final : public RTCEncodedFrameBase {
 
   bool IsVideo() const override { return false; }
 
+  // [Serializable] implementations: {Read, Write}StructuredClone
+  static already_AddRefed<RTCEncodedAudioFrame> ReadStructuredClone(
+      JSContext* aCx, nsIGlobalObject* aGlobal,
+      JSStructuredCloneReader* aReader);
+  bool WriteStructuredClone(JSContext* aCx,
+                            JSStructuredCloneWriter* aWriter) const;
+
  private:
   virtual ~RTCEncodedAudioFrame();
+
+  // forbid copy/move to keep mState member in base valid
+  RTCEncodedAudioFrame(const RTCEncodedAudioFrame&) = delete;
+  RTCEncodedAudioFrame& operator=(const RTCEncodedAudioFrame&) = delete;
+  RTCEncodedAudioFrame(RTCEncodedAudioFrame&&) = delete;
+  RTCEncodedAudioFrame& operator=(RTCEncodedAudioFrame&&) = delete;
+
   RefPtr<RTCRtpScriptTransformer> mOwner;
-  RTCEncodedAudioFrameMetadata mMetadata;
 };
 
 }  // namespace mozilla::dom

dom/media/webrtc/jsapi/RTCEncodedFrameBase.cpp

@@ -6,6 +6,7 @@
 
 #include "jsapi/RTCEncodedFrameBase.h"
 
+#include "api/frame_transformer_interface.h"
 #include "js/ArrayBuffer.h"
 #include "js/GCAPI.h"
 #include "mozilla/dom/ScriptSettings.h"
@@ -21,53 +22,63 @@ NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(RTCEncodedFrameBase)
   NS_INTERFACE_MAP_ENTRY(nsISupports)
 NS_INTERFACE_MAP_END
 
-RTCEncodedFrameBase::RTCEncodedFrameBase(
-    nsIGlobalObject* aGlobal,
-    std::unique_ptr<webrtc::TransformableFrameInterface> aFrame,
-    uint64_t aCounter)
-    : mGlobal(aGlobal),
-      mFrame(std::move(aFrame)),
-      mCounter(aCounter),
-      mTimestamp(mFrame->GetTimestamp()) {
+RTCEncodedFrameBase::RTCEncodedFrameBase(nsIGlobalObject* aGlobal,
+                                         RTCEncodedFrameState& aState)
+    : mGlobal(aGlobal), mState(aState), mData(nullptr) {
+  mState.mTimestamp = mState.mFrame->GetTimestamp();
   AutoJSAPI jsapi;
   if (NS_WARN_IF(!jsapi.Init(mGlobal))) {
     return;
   }
 
   // Avoid a copy
   mData = JS::NewArrayBufferWithUserOwnedContents(
-      jsapi.cx(), mFrame->GetData().size(), (void*)(mFrame->GetData().data()));
+      jsapi.cx(), mState.mFrame->GetData().size(),
+      (void*)(mState.mFrame->GetData().data()));
 }
 
 RTCEncodedFrameBase::~RTCEncodedFrameBase() = default;
 
-unsigned long RTCEncodedFrameBase::Timestamp() const { return mTimestamp; }
+unsigned long RTCEncodedFrameBase::Timestamp() const {
+  return mState.mTimestamp;
+}
 
 void RTCEncodedFrameBase::SetData(const ArrayBuffer& aData) {
   mData.set(aData.Obj());
-  if (mFrame) {
+  if (mState.mFrame) {
     aData.ProcessData([&](const Span<uint8_t>& aData, JS::AutoCheckCannotGC&&) {
-      mFrame->SetData(
+      mState.mFrame->SetData(
           webrtc::ArrayView<const uint8_t>(aData.Elements(), aData.Length()));
     });
   }
 }
 
-void RTCEncodedFrameBase::GetData(JSContext* aCx, JS::Rooted<JSObject*>* aObj) {
+void RTCEncodedFrameBase::GetData(JSContext* aCx,
+                                  JS::Rooted<JSObject*>* aObj) const {
   aObj->set(mData);
 }
 
-uint64_t RTCEncodedFrameBase::GetCounter() const { return mCounter; }
+uint64_t RTCEncodedFrameBase::GetCounter() const { return mState.mCounter; }
 
 std::unique_ptr<webrtc::TransformableFrameInterface>
 RTCEncodedFrameBase::TakeFrame() {
-  AutoJSAPI jsapi;
-  if (!jsapi.Init(mGlobal)) {
-    MOZ_CRASH("Could not init JSAPI!");
+  if (mState.mFrame) {
+    AutoJSAPI jsapi;
+    if (!jsapi.Init(mGlobal)) {
+      MOZ_CRASH("Could not init JSAPI!");
+    }
+    // If the JS buffer was transferred (or otherwise detached), neuter native.
+    JS::Rooted<JSObject*> rootedData(jsapi.cx(), mData);
+    if (rootedData && JS::IsDetachedArrayBufferObject(rootedData)) {
+      mState.mFrame.reset();
+      return nullptr;
+    }
+    // Still attached: detach now since we're consuming the frame.
+    JS::DetachArrayBuffer(jsapi.cx(), rootedData);
   }
-  JS::Rooted<JSObject*> rootedData(jsapi.cx(), mData);
-  JS::DetachArrayBuffer(jsapi.cx(), rootedData);
-  return std::move(mFrame);
+  return std::move(mState.mFrame);
 }
 
+RTCEncodedFrameState::~RTCEncodedFrameState() = default;
+
 }  // namespace mozilla::dom

dom/media/webrtc/jsapi/RTCEncodedFrameBase.h

@@ -9,23 +9,40 @@
 
 #include <memory>
 
-#include "api/frame_transformer_interface.h"
 #include "js/TypeDecls.h"
 #include "mozilla/Assertions.h"
 #include "mozilla/dom/TypedArray.h"  // ArrayBuffer
 
 class nsIGlobalObject;
 
+namespace webrtc {
+class TransformableFrameInterface;
+}
+
 namespace mozilla::dom {
 
+struct RTCEncodedFrameState {
+  std::unique_ptr<webrtc::TransformableFrameInterface> mFrame;
+  uint64_t mCounter = 0;
+  unsigned long mTimestamp = 0;
+
+  // work around only having forward-declared TransformableFrameInterface
+  ~RTCEncodedFrameState();
+
+  // avoid "move got disabled by a user-declared destructor” trap
+  RTCEncodedFrameState() = default;
+  RTCEncodedFrameState(RTCEncodedFrameState&&) noexcept = default;
+  RTCEncodedFrameState& operator=(RTCEncodedFrameState&&) noexcept = default;
+  RTCEncodedFrameState(const RTCEncodedFrameState&) = delete;
+  RTCEncodedFrameState& operator=(const RTCEncodedFrameState&) = delete;
+};
+
 class RTCRtpScriptTransformer;
 
 class RTCEncodedFrameBase : public nsISupports, public nsWrapperCache {
  public:
-  explicit RTCEncodedFrameBase(
-      nsIGlobalObject* aGlobal,
-      std::unique_ptr<webrtc::TransformableFrameInterface> aFrame,
-      uint64_t aCounter);
+  explicit RTCEncodedFrameBase(nsIGlobalObject* aGlobal,
+                               RTCEncodedFrameState& aState);
 
   // nsISupports
   NS_DECL_CYCLE_COLLECTING_ISUPPORTS
@@ -36,7 +53,7 @@ class RTCEncodedFrameBase : public nsISupports, public nsWrapperCache {
 
   void SetData(const ArrayBuffer& aData);
 
-  void GetData(JSContext* aCx, JS::Rooted<JSObject*>* aObj);
+  void GetData(JSContext* aCx, JS::Rooted<JSObject*>* aObj) const;
 
   uint64_t GetCounter() const;
 
@@ -48,10 +65,20 @@ class RTCEncodedFrameBase : public nsISupports, public nsWrapperCache {
 
  protected:
   virtual ~RTCEncodedFrameBase();
+
+  // forbid copy/move to protect mState
+  RTCEncodedFrameBase(const RTCEncodedFrameBase&) = delete;
+  RTCEncodedFrameBase& operator=(const RTCEncodedFrameBase&) = delete;
+  RTCEncodedFrameBase(RTCEncodedFrameBase&&) = delete;
+  RTCEncodedFrameBase& operator=(RTCEncodedFrameBase&&) = delete;
+
   RefPtr<nsIGlobalObject> mGlobal;
-  std::unique_ptr<webrtc::TransformableFrameInterface> mFrame;
-  const uint64_t mCounter = 0;
-  const unsigned long mTimestamp = 0;
+
+  // Keep serializable state separate in this base and its subclasses
+  // in a manner that avoids diamond inheritance. Subclasses must pass
+  // in *this, to ensure it's constructed before and destroyed after
+  // this base; copy and move are deleted.
+  RTCEncodedFrameState& mState;
   JS::Heap<JSObject*> mData;
 };