Add video background effects and blur to participant video feeds in your RealtimeKit meetings using the Core SDK.
npm i @cloudflare/realtimekit-virtual-backgroundyarn add @cloudflare/realtimekit-virtual-backgroundpnpm add @cloudflare/realtimekit-virtual-backgroundbun add @cloudflare/realtimekit-virtual-backgroundDisable the default per frame rendering of video middleware to improve speed and quality by letting this middleware control it on its own:
await meeting.self.setVideoMiddlewareGlobalConfig({
disablePerFrameCanvasRendering: true,
});Create a video background transformer object:
import RealtimeKitVideoBackgroundTransformer from "@cloudflare/realtimekit-virtual-background";
const videoBackgroundTransformer =
await RealtimeKitVideoBackgroundTransformer.init({
meeting,
});The videoBackgroundTransformer exposes two types of middlewares:
Use createStaticBackgroundVideoMiddleware to set an image as the background:
const imageUrl = "https://images.unsplash.com/photo-1487088678257-3a541e6e3922";
meeting.self.addVideoMiddleware(
await videoBackgroundTransformer.createStaticBackgroundVideoMiddleware(
imageUrl,
),
);Use createBackgroundBlurVideoMiddleware to blur the background. Pass blurStrength (0-100) as a parameter (50% by default):
meeting.self.addVideoMiddleware(
await videoBackgroundTransformer.createBackgroundBlurVideoMiddleware(50),
);Check browser support before initializing:
if (RealtimeKitVideoBackgroundTransformer.isSupported()) {
const videoBackgroundTransformer =
await RealtimeKitVideoBackgroundTransformer.init({
meeting: meeting,
});
meeting.self.addVideoMiddleware(
await videoBackgroundTransformer.createStaticBackgroundVideoMiddleware(
imageUrl,
),
);
}For better, sharper results, pass a custom segmentation configuration:
const videoBackgroundTransformer =
await RealtimeKitVideoBackgroundTransformer.init({
meeting,
segmentationConfig: {
model: "mlkit", // 'meet' | 'mlkit'
backend: "wasmSimd",
inputResolution: "256x256", // '256x144' for meet
pipeline: "webgl2", // 'webgl2' | 'canvas2dCpu'
// canvas2dCpu gives sharper blur, webgl2 is faster
targetFps: 35,
},
});npm i @cloudflare/realtimekit-virtual-backgroundyarn add @cloudflare/realtimekit-virtual-backgroundpnpm add @cloudflare/realtimekit-virtual-backgroundbun add @cloudflare/realtimekit-virtual-backgroundimport { useState, useEffect } from "react";
import { useRealtimeKitClient } from "@cloudflare/realtimekit-react";
import RealtimeKitVideoBackgroundTransformer from "@cloudflare/realtimekit-virtual-background";
function App() {
const [meeting] = useRealtimeKitClient();
const [videoBackgroundTransformer, setVideoBackgroundTransformer] =
useState(null);
useEffect(() => {
const initializeTransformer = async () => {
if (!meeting) return;
// Check browser support
if (!RealtimeKitVideoBackgroundTransformer.isSupported()) {
console.warn("Video background not supported in this browser");
return;
}
// Disable default per frame rendering
await meeting.self.setVideoMiddlewareGlobalConfig({
disablePerFrameCanvasRendering: true,
});
// Initialize transformer
const transformer = await RealtimeKitVideoBackgroundTransformer.init({
meeting,
});
setVideoBackgroundTransformer(transformer);
};
initializeTransformer();
}, [meeting]);
const applyStaticBackground = async (imageUrl) => {
if (!videoBackgroundTransformer) return;
meeting.self.addVideoMiddleware(
await videoBackgroundTransformer.createStaticBackgroundVideoMiddleware(
imageUrl,
),
);
};
const applyBlur = async (blurStrength = 50) => {
if (!videoBackgroundTransformer) return;
meeting.self.addVideoMiddleware(
await videoBackgroundTransformer.createBackgroundBlurVideoMiddleware(
blurStrength,
),
);
};
const removeBackground = () => {
// Remove all video middlewares
meeting.self.removeVideoMiddleware();
};
return (
<div>
<button
onClick={() =>
applyStaticBackground(
"https://images.unsplash.com/photo-1487088678257-3a541e6e3922",
)
}
>
Apply Background
</button>
<button onClick={() => applyBlur(50)}>Apply Blur</button>
<button onClick={removeBackground}>Remove Background</button>
</div>
);
}For better, sharper results, pass a custom segmentation configuration:
const transformer = await RealtimeKitVideoBackgroundTransformer.init({
meeting,
segmentationConfig: {
model: "mlkit", // 'meet' | 'mlkit'
backend: "wasmSimd",
inputResolution: "256x256", // '256x144' for meet
pipeline: "webgl2", // 'webgl2' | 'canvas2dCpu'
// canvas2dCpu gives sharper blur, webgl2 is faster
targetFps: 35,
},
});npm i @cloudflare/realtimekit-virtual-backgroundyarn add @cloudflare/realtimekit-virtual-backgroundpnpm add @cloudflare/realtimekit-virtual-backgroundbun add @cloudflare/realtimekit-virtual-backgroundIn your component TypeScript file:
import { Component, OnInit } from "@angular/core";
import RealtimeKitClient from "@cloudflare/realtimekit";
import RealtimeKitVideoBackgroundTransformer from "@cloudflare/realtimekit-virtual-background";
@Component({
selector: "app-meeting",
templateUrl: "./meeting.component.html",
})
export class MeetingComponent implements OnInit {
meeting: any;
videoBackgroundTransformer: any;
async ngOnInit() {
// Initialize meeting
this.meeting = await RealtimeKitClient.init({
authToken: "<participant_auth_token>",
});
await this.meeting.join();
// Check browser support
if (!RealtimeKitVideoBackgroundTransformer.isSupported()) {
console.warn("Video background not supported in this browser");
return;
}
// Disable default per frame rendering
await this.meeting.self.setVideoMiddlewareGlobalConfig({
disablePerFrameCanvasRendering: true,
});
// Initialize transformer
this.videoBackgroundTransformer =
await RealtimeKitVideoBackgroundTransformer.init({
meeting: this.meeting,
});
}
async applyStaticBackground(imageUrl: string) {
if (!this.videoBackgroundTransformer) return;
this.meeting.self.addVideoMiddleware(
await this.videoBackgroundTransformer.createStaticBackgroundVideoMiddleware(
imageUrl,
),
);
}
async applyBlur(blurStrength: number = 50) {
if (!this.videoBackgroundTransformer) return;
this.meeting.self.addVideoMiddleware(
await this.videoBackgroundTransformer.createBackgroundBlurVideoMiddleware(
blurStrength,
),
);
}
removeBackground() {
// Remove all video middlewares
this.meeting.self.removeVideoMiddleware();
}
}In your component template:
<button
(click)="applyStaticBackground('https://images.unsplash.com/photo-1487088678257-3a541e6e3922')"
>
Apply Background
</button>
<button (click)="applyBlur(50)">Apply Blur</button>
<button (click)="removeBackground()">Remove Background</button>For better, sharper results, pass a custom segmentation configuration:
this.videoBackgroundTransformer =
await RealtimeKitVideoBackgroundTransformer.init({
meeting: this.meeting,
segmentationConfig: {
model: "mlkit", // 'meet' | 'mlkit'
backend: "wasmSimd",
inputResolution: "256x256", // '256x144' for meet
pipeline: "webgl2", // 'webgl2' | 'canvas2dCpu'
// canvas2dCpu gives sharper blur, webgl2 is faster
targetFps: 35,
},
});You can add the pre-packaged filters to your project by adding the following dependency to your build.gradle file:
dependencies {
// (other dependencies)
implementation 'com.cloudflare.realtimekit:filters:0.1.0'
}This package currently exposes VirtualBackgroundVideoFilter which can be used with FilterVideoProcessor:
// Create a virtual background filter with a custom background image.
val bgFilter = VirtualBackgroundVideoFilter(context, R.drawable.background)
// Initialize the video processor with the filter.
val processor = FilterVideoProcessor(eglBase, bgFilter)
// // Set the video processor on the meeting builder.
val meeting = RealtimeKitMeetingBuilder
.setVideoProcessor(eglBase, processor)
.build(activity)You can also create your own custom filters to apply effects, filters, or analytics directly to a live video stream. Our VideoProcessor APIs provide flexible and powerful ways to manipulate video frames.
We provide three types of video processors:
- NoDropVideoProcessor: Allows custom video processing without dropping frames.
- ChainVideoProcessor: Chains multiple frame processors together, useful for applying multiple effects or filters to a video stream.
- FilterVideoProcessor: Simpler and more efficient way to apply a single effect or filter to a video stream.
Nonetheless, you can also create your own custom video processors by implementing the VideoProcessor interface directly:
import realtimekit.org.webrtc.VideoFrame
import realtimekit.org.webrtc.VideoProcessor
import realtimekit.org.webrtc.VideoSink
class CustomVideoProcessor : VideoProcessor {
override fun onCapturerStarted(started: Boolean) {}
override fun onCapturerStopped() {}
override fun onFrameCaptured(frame: VideoFrame?) {}
override fun setSink(sink: VideoSink?) {}
}Once you have created and configured your VideoProcessor, pass it to the RealtimeKitMeetingBuilder object. This will process video frames captured by the camera before they are sent to other participants or rendered locally:
// Assuming 'myCustomProcessor' is an instance of any VideoProcessor implementation
// (for example, ChainVideoProcessor, FilterVideoProcessor, and more).
val myCustomProcessor = CustomProcessor()
// Set the video processor on the meeting builder.
val meeting = RealtimeKitMeetingBuilder
.setVideoProcessor(processor = myCustomProcessor)
.build(activity)
// You can also pass an EglBase to the builder
// This is useful when using FilterVideoProcessor
val eglBase = EglBase.create()
val meeting = RealtimeKitMeetingBuilder
.setVideoProcessor(eglBase = eglBase, processor = myCustomProcessor)
.build(activity)