Technical background
Unity3D can be used to create various types of applications, including virtual reality, training simulators, etc. Here are some scenes that can be played using Unity3D panorama:
Virtual reality experience: Panoramic video can be used to create a realistic virtual environment, allowing users to feel immersed in the scene;
Training simulator: Panoramic video can be used to create real training environments, such as flight simulators, driving simulators, etc., to provide a more realistic training experience;
Architectural design: Panoramic video can be used to show architectural design or interior decoration, so that customers can feel the real effect;
Cultural tourism guide: Panoramic videos can be used to show tourist attractions or cities, allowing tourists to feel immersed in the scene.
To achieve panoramic real-time RTMP or RTSP stream rendering on the Unity3D platform, you can use the following methods:
- Obtain the panoramic video data source: First, you need to pull the RTMP or RTSP stream data. After decoding, call the RGB or YUV data back to unity to obtain the panoramic video stream data;
- Unity creates a Sphere, creates a material ball (Material), and hangs the material ball on the Sphere;
- Achieve real-time rendering: Using Unity3D’s rendering pipeline, you can map textures onto the surface of a sphere or cube and use shaders to process the coordinates of the texture to achieve real-time rendering of panoramic videos.
Technical realization
This article uses the RTMP push end of Daniu Live SDK as data collection. After obtaining the panoramic form data, encode and package it and push it to the RTMP service, or start a lightweight RTSP service and provide an RTSP streaming URL to the outside world.
Then, on the player side, pull the RTSP or RTMP URL, call back the YUV or RGB data, and then draw it in the Unity form.
Get the data source:
public void Play(int sel)
{
if (videoctrl[sel].is_running)
{
Debug.Log("Already playing..");
return;
}
lock (videoctrl[sel].frame_lock_)
{
videoctrl[sel].cur_video_frame_ = null;
}
OpenPlayer(sel);
if (videoctrl[sel].player_handle_ == IntPtr.Zero)
return;
//Set playback URL
NTSmartPlayerSDK.NT_SP_SetURL(videoctrl[sel].player_handle_, videoctrl[sel].videoUrl);
/* + + Parameter configuration before playback can be added here + + */
int play_buffer_time_ = 0;
NTSmartPlayerSDK.NT_SP_SetBuffer(videoctrl[sel].player_handle_, play_buffer_time_); //Set buffer time
int is_using_tcp = 0; //TCP mode
NTSmartPlayerSDK.NT_SP_SetRTSPTcpMode(videoctrl[sel].player_handle_, is_using_tcp);
int timeout = 10;
NTSmartPlayerSDK.NT_SP_SetRtspTimeout(videoctrl[sel].player_handle_, timeout);
int is_auto_switch_tcp_udp = 1;
NTSmartPlayerSDK.NT_SP_SetRtspAutoSwitchTcpUdp(videoctrl[sel].player_handle_, is_auto_switch_tcp_udp);
Boolean is_mute_ = false;
NTSmartPlayerSDK.NT_SP_SetMute(videoctrl[sel].player_handle_, is_mute_ ? 1 : 0); //Whether to mute when starting playback
int is_fast_startup = 1;
NTSmartPlayerSDK.NT_SP_SetFastStartup(videoctrl[sel].player_handle_, is_fast_startup); //Set fast startup mode
Boolean is_low_latency_ = false;
NTSmartPlayerSDK.NT_SP_SetLowLatencyMode(videoctrl[sel].player_handle_, is_low_latency_ ? 1 : 0); //Set whether to enable low latency mode
//Set the rotation angle (settings of 0, 90, 180, and 270 degrees are valid, other values are invalid)
int rotate_degrees = 0;
NTSmartPlayerSDK.NT_SP_SetRotation(videoctrl[sel].player_handle_, rotate_degrees);
\t\t
int volume = 100;
NTSmartPlayerSDK.NT_SP_SetAudioVolume(videoctrl[sel].player_handle_, volume); //Set the playback volume, the range is [0, 100], 0 is mute, 100 is the maximum volume, the default is 100
\t\t
//Set the upload and download speed
int is_report = 0;
int report_interval = 2;
NTSmartPlayerSDK.NT_SP_SetReportDownloadSpeed(videoctrl[sel].player_handle_, is_report, report_interval);
/* -- Parameter configuration before playback can be added here -- */
//video frame callback (YUV/RGB)
videoctrl[sel].video_frame_call_back_ = new SP_SDKVideoFrameCallBack(NT_SP_SetVideoFrameCallBack);
NTSmartPlayerSDK.NT_SP_SetVideoFrameCallBack(videoctrl[sel].player_handle_, (Int32)NT.NTSmartPlayerDefine.NT_SP_E_VIDEO_FRAME_FORMAT.NT_SP_E_VIDEO_FRAME_FROMAT_I420, window_handle_, videoctrl[sel].video_frame_call_back_);
UInt32 flag = NTSmartPlayerSDK.NT_SP_StartPlay(videoctrl[sel].player_handle_);
if (flag == DANIULIVE_RETURN_OK)
{
videoctrl[sel].is_need_get_frame_ = true;
Debug.Log("Playback successful");
}
else
{
videoctrl[sel].is_need_get_frame_ = false;
Debug.LogError("Playback failed");
}
videoctrl[sel].is_running = true;
}
For data processing:
private void SDKVideoFrameCallBack(UInt32 status, IntPtr frame, int sel)
{
//Get the callback frame here and perform related operations
NT_SP_VideoFrame video_frame = (NT_SP_VideoFrame)Marshal.PtrToStructure(frame, typeof(NT_SP_VideoFrame));
VideoFrame u3d_frame = new VideoFrame();
u3d_frame.width_ = video_frame.width_;
u3d_frame.height_ = video_frame.height_;
u3d_frame.timestamp_ = (UInt64)video_frame.timestamp_;
int d_y_stride = video_frame.width_;
int d_u_stride = (video_frame.width_ + 1) / 2;
int d_v_stride = d_u_stride;
int d_y_size = d_y_stride * video_frame.height_;
int d_u_size = d_u_stride * ((video_frame.height_ + 1) / 2);
int d_v_size = d_u_size;
int u_v_height = ((u3d_frame.height_ + 1) / 2);
u3d_frame.y_stride_ = d_y_stride;
u3d_frame.u_stride_ = d_u_stride;
u3d_frame.v_stride_ = d_v_stride;
u3d_frame.y_data_ = new byte[d_y_size];
u3d_frame.u_data_ = new byte[d_u_size];
u3d_frame.v_data_ = new byte[d_v_size];
CopyFramePlane(u3d_frame.y_data_, d_y_stride,
video_frame.plane0_, video_frame.stride0_, u3d_frame.height_);
CopyFramePlane(u3d_frame.u_data_, d_u_stride,
video_frame.plane1_, video_frame.stride1_, u_v_height);
CopyFramePlane(u3d_frame.v_data_, d_v_stride,
video_frame.plane2_, video_frame.stride2_, u_v_height);
lock (videoctrl[sel].frame_lock_ )
{
videoctrl[sel].cur_video_frame_ = u3d_frame;
}
}
Refresh Texture:
private void UpdateYUVTexture(VideoFrame video_frame, int sel)
{
if (video_frame.y_data_ == null || video_frame.u_data_ == null || video_frame.v_data_ == null)
{
Debug.Log("video frame with null..");
return;
}
if (videoctrl[sel].yTexture_ != null)
{
videoctrl[sel].yTexture_.LoadRawTextureData(video_frame.y_data_);
videoctrl[sel].yTexture_.Apply();
}
if (videoctrl[sel].uTexture_ != null)
{
videoctrl[sel].uTexture_.LoadRawTextureData(video_frame.u_data_);
videoctrl[sel].uTexture_.Apply();
}
if (videoctrl[sel].vTexture_ != null)
{
videoctrl[sel].vTexture_.LoadRawTextureData(video_frame.v_data_);
videoctrl[sel].vTexture_.Apply();
}
}
During panoramic playback, if you need to move the display area, you can use the following code:
public class MouseMove : MonoBehaviour {
Vector2 p1, p2;
private Vector3 PreMouseMPos;
private Vector3 PreMouseLPos;
public float minimumY = -60F;
public float maximumY = 60F;
float rotationY = 0F;
private float wheelSpeed = 5.0f;
\t
// Use this for initialization
void Start () {
}
\t
// Update is called once per frame
void Update () {
if (Input.GetMouseButton(0))
{
if (PreMouseLPos.x <= 0)
{
PreMouseLPos = new Vector3(Input.mousePosition.x, Input.mousePosition.y, 0.0f);
}
else
{
Vector3 CurMouseLPos = new Vector3(Input.mousePosition.x, Input.mousePosition.y, 0.0f);
Vector3 offset = CurMouseLPos - PreMouseLPos;
Quaternion tt = Quaternion.Euler(offset);
float rotationX = transform.localEulerAngles.y - tt.x * 20;
rotationY + = tt.y * 20;
rotationY = Mathf.Clamp(rotationY, minimumY, maximumY);
transform.localEulerAngles = new Vector3(rotationY, rotationX, 0);
PreMouseLPos = CurMouseLPos;
}
}
else
{
PreMouseLPos = new Vector3(0.0f, 0.0f, 0.0f);
}
}
}
Summarize
Unity panoramic playback of RTMP or RTSP real-time streams can be widely used in various scenarios that need to provide real scenes or immersive experiences, bringing users a more realistic experience. At the same time, Unity panoramic real-time playback requires very high latency and performance requirements, especially the panoramic data source, which has very high resolution and bit rate, which improves decoding efficiency and decoded data copy delivery. requirements.
Author: Audio and Video Niu Brother
Original article How to achieve low-latency panoramic RTMP|RTSP stream rendering under Unity – Nuggets
On the business card at the end of the article, you can get free audio and video development learning materials, including (FFmpeg, webRTC, rtmp, hls, rtsp, ffplay, srs) and audio and video learning roadmap, etc.
See below! ↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓