rtsp decoding
Use live555, receive in a thread
class c_rtspthread:public c_thread
{
int v_headlen = 0;
c_rtsp *v_rtsp = nullptr;
//32-bit hash value
uint32_t v_key = 0;//hash(live/1001);
uint32_t _recv_stamp = 0;
uint32_t _first_stamp = 0;
sp_buffer _spbuffer;
c_flvserver *v_flv;
std::string v_livename;//live/1001
private:
//decode use it
AVCodec *v_codec = NULL;
AVCodecContext *v_codecctx = NULL;
AVFrame *v_frame = NULL;
c_analyse *v_analyse = NULL;
int do_decode_init(const char *name,const char *codec);
int do_decode_unit();
int width()
{
if (v_codecctx != NULL)
return v_codecctx->width;
return 0;
}
int height()
{
if (v_codecctx != NULL)
return v_codecctx->height;
return 0;
}
int v_width = 0;
int v_height= 0;
int v_fps = 0;
int v_towidth = 0;
int v_toheight = 0;
int decode2YUV(uint8_t* src,
int srcLen,
uint8_t *destYuv,
int destw, int desth);
void decode2RGB(uint8_t* src, int & srcLen);
struct SwsContext *_img_convert_ctx = NULL;
public:
void init_start(c_flvserver * flv, const char * url, const char* livename, int towidth, int toheight, uint32_t key);
int callback(const char* flag, uint8_t * data,long size, uint32_t ts);
//Rewrite stop function
void Stop();
//Disconnect and reconnect
void Run();
};
Analysis using opencv
In order to use the opencv used by the public, opencv is called directly here. The python call needs to be considered. Combine the avframe of ffmepg with the mat of opencv, mainly the following sentences
AVFrame *dframe = av_frame_alloc();
cv::Mat nmat;
nmat.create(cv::Size(w, h), CV_8UC3);
//printf("frame =\\
", v_codecctx->frame_number);
av_image_fill_arrays(dframe->data, dframe->linesize, nmat.data, AV_PIX_FMT_BGR24,
w, h, 16);
When decoding, directly associate cv::Mat with the memory of AVFrame, do not copy it back and forth. In fact, opencv’s Mat is generally in BGR mode. If you need a grayscale image, just decode it directly into YUV and then take the Y component.
int c_rtspthread::decode2YUV(uint8_t* src, int srcLen, uint8_t *destYuv, int destw, int desth)
{
cv::Mat m;// (Width, Height, CV_8UC1);
//int gotPicture = 0;
AVPacket pkt;
av_init_packet( & amp;pkt);
pkt.data = src;
pkt.size = srcLen;
int ret = avcodec_send_packet(v_codecctx, & amp;pkt);
av_packet_unref( & amp;pkt);
if (ret < 0)
{
fprintf(stderr, "Error sending a packet for decoding\\
");
return -1;
}
//fixme:qianbo maybe receive more frame;
while (ret >= 0) {
AVFrame *frame = av_frame_alloc();
ret = avcodec_receive_frame(v_codecctx, frame);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
{
av_frame_free( & amp;frame);
return 0;
}
else if (ret < 0) {
av_frame_free( & amp;frame);
//fprintf(stderr, "Error during decoding\\
");
return 0;
}
//printf("frame =\\
", v_codecctx->frame_number);
if (v_analyse != NULL)
{
v_analyse->pushdata2(frame);
}
}
#if 0
if (_img_convert_ctx == NULL)
{
if (v_destframe == NULL)
v_destframe = av_frame_alloc();
if (destw == 0)
destw = Width;
if (desth == 0)
desth = Height;
av_image_fill_arrays(v_destframe->data, v_destframe->linesize, destYuv, AV_PIX_FMT_YUV420P, destw, desth, 1);
_img_convert_ctx = sws_getContext(Width, Height,
_codecCtx->pix_fmt,//PIX_FMT_YUV420P,
destw,
Desth,
AV_PIX_FMT_YUV420P,
SWS_POINT,
//SWS_BICUBIC,
NULL,
NULL,
NULL);
}
sws_scale(_img_convert_ctx, _Frame->data, _Frame->linesize, 0, Height, _yuvFrame->data, _yuvFrame->linesize);
#endif
return -1;
}
void c_rtspthread::decode2RGB(uint8_t* src, int & srcLen)
{
AVPacket pkt;
av_init_packet( & amp;pkt);
pkt.data = src;
pkt.size = srcLen;
int ret = avcodec_send_packet(v_codecctx, & amp;pkt) == 0;
av_packet_unref( & amp;pkt);
if (ret < 0)
{
fprintf(stderr, "Error sending a packet for decoding\\
");
return;
}
while (ret >= 0)
{
\t\t
ret = avcodec_receive_frame(v_codecctx, v_frame);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
{
//av_frame_free( & amp;frame);
break;
}
else if (ret < 0) {
//fprintf(stderr, "Error during decoding\\
");
//av_frame_free( & amp;frame);
break;
}
int w = v_towidth; //v_frame->width;
int h = v_toheight; //v_frame->height;
if (_img_convert_ctx == NULL)
{
_img_convert_ctx = sws_getContext(v_frame->width, v_frame->height,
v_codecctx->pix_fmt/*AV_PIX_FMT_YUV420P*/,
w,
h,
AV_PIX_FMT_BGR24,
//SWS_POINT,
SWS_BICUBIC,
NULL,
NULL,
NULL);
}
AVFrame *dframe = av_frame_alloc();
cv::Mat nmat;
nmat.create(cv::Size(w, h), CV_8UC3);
//printf("frame =\\
", v_codecctx->frame_number);
av_image_fill_arrays(dframe->data, dframe->linesize, nmat.data, AV_PIX_FMT_BGR24,
w, h, 16);
sws_scale(_img_convert_ctx, v_frame->data, v_frame->linesize, 0,
v_frame->height,
dframe->data, dframe->linesize);
if (v_analyse != NULL)
{
v_analyse->pushdata(nmat);
}
av_frame_free( & amp;dframe);
}
//av_packet_unref( & amp;pkt);
}
forward flv
This part can be divided into two ways, one is to send it directly to the existing flvserver, and the other is to directly become a flvserver yourself. In terms of efficiency, directly becoming a flvserver can be the preferred option. First, use the coroutine of the boost library to make an httpserver. , because websocketserver is built on httpserver
class c_http_session:public std::enable_shared_from_this<c_http_session>
{
public:
uint32_t v_key = 0;
uint32_t v_start_ts = 0;
tcp::socket v_socket;
int v_has_send_meta = 0;
int v_has_send_video = 0;
int v_has_send_audio = 0;
int v_has_sent_key_frame = 0;
asio::strand<asio::io_context::executor_type> v_strand;
void close()
{
if (v_socket.is_open())
v_socket.close();
/*if (v_key > 0)
c_flvhubs::instance()->pop(v_key, shared_from_this());*/
}
public:
bool func_hand_shake(boost::asio::yield_context & amp;yield)
{
return false;
}
void go()
{
auto self(shared_from_this());
boost::asio::spawn(v_strand,
[this, self](boost::asio::yield_context yield)
{
//try
//{
//timer_.expires_from_now(std::chrono::seconds(10));
if (func_hand_shake(yield) == false)
{
std::cout << "not hand shake" << std::endl;
return;
}
for (;;)
{
//bool ret = func_recv_message(yield);
/*if (!ret)
{
close();
break;
}*/
}
//}
//catch (std::exception & amp; e)
//{
// std::cout << "some is error:" << e.what() << std::endl;
// close();
// //timer_.cancel();
//}
});
}
};
The above class is not actually used, because after writing the websocket server, the situation of httpserver must also be taken into consideration. In fact, the data volume of httpsever is smaller than that of websocket, except for the beginning header, because websocketserver must change the size of the frame data every time. Sending back to the peer actually solves the sticky problem of TCP, but in return, the header of flv also has this data length, so http flv can send data directly.
**According to the RFC document 6455 document, **With a clear understanding of the principles and headers, you can make a simple websocket server. Note that the data sent by the browser is encrypted and needs to be decoded here because the whole process is the same as browsing. Interactive with the server, so it is easy to debug. Write a javascript for debugging, as follows:
This html can receive images returned from the server. As a tool, it can be used for debugging.
<!DOCTYPE HTML>
<html>
<head>
<meta charset="utf-8">
<title></title>
</head>
<body>
<div id="imgDiv"></div>
<div id="sse">
<a href="javascript:WebSocketTest()">Run WebSocket</a>
</div>
<script type="text/javascript">
function init() {
canvas = document.createElement('canvas');
content = canvas.getContext('2d');
canvas.width = 320;
canvas.height = 240;
content.scale(1, -1);
content.translate(0, -240);
document.body.appendChild(canvas);
// container.appendChild(canvas);
img = new Image();
img.src = "bg1.jpg";
canvas.style.position = 'absolute';
img.onload = function () {
content.drawImage(img, 0, 0, canvas.width, canvas.height);
//URL.revokeObjectURL(url);
// imgDate = content.getImageData(0, 0, canvas.width, canvas.height);
//createPotCloud(); //Create point cloud
};
}
init();
function WebSocketTest() {
if ("WebSocket" in window) {
// alert("Your browser supports WebSocket!");
//Open a web socket
var ws = new WebSocket("ws://127.0.0.1:9000/live/image");
console.log(ws);
ws.onopen = function (evt) {
console.log("connected");
/*let obj = JSON.stringify({
test:"qianbo0423"
})
ws.send(obj);*/
};
ws.onmessage = function (evt) {
if (typeof (evt.data) == "string") {
//textHandler(JSON.parse(evt.data));
} else {
var reader = new FileReader();
reader.onload = function (evt) {
if (evt.target.readyState == FileReader.DONE) {
var url = evt.target.result;
// console.log(url);
img.src = url;
//img.src = url;// "bg1.jpg";
//var imga = document.getElementById("imgDiv");
//imga.innerHTML = "<img src = " + url + " />";
}
}
reader.readAsDataURL(evt.data);
}
};
ws.onclose = function () {
alert("The connection has been closed...");
};
} else {
// The browser does not support WebSocket
alert("Your browser does not support WebSocket!");
}
}
</script>
</body>
</html>
Secondly, in addition to receiving rtsp, you can also directly receive the canvas data sent by the browser, use ws to receive it, and then use the analysis function to parse out the encoded file or h264 data. The following is the html canvas sent to the websocket server.
<!doctype html>
<html>
<head>
<meta charset="utf-8">
<script src="js/canvas.js"></script>
<script>
document.addEventListener('DOMContentLoaded', () => {
document.querySelector('[data-action="goLive"]').addEventListener('click', (e) => {
//console.log(createRes);
let mediaRecorder;
let mediaStream;
var addr = "ws://127.0.0.1:3000/1001";
const ws = new WebSocket(addr);
ws.addEventListener('open', (e) => {
console.log('WebSocket Open', e);
mediaStream = document.querySelector('canvas').captureStream(20); // 10 FPS
mediaRecorder = new MediaRecorder(mediaStream, {
mimeType: 'video/webm;codecs=h264',
videoBitsPerSecond: 500000
});
mediaRecorder.addEventListener('dataavailable', (e) => {
console.log(e.data);
ws.send(e.data);
});
mediaRecorder.addEventListener('stop', ws.close.bind(ws));
mediaRecorder.start(500); // Start recording, and dump data every second
});
ws.addEventListener('close', (e) => {
console.log('WebSocket Close', e);
mediaRecorder.stop();
});
});
});
</script>
</head>
<body>
<canvas width="640" height="360"></canvas>
<nav>
<button data-action="Transfer">Transfer</button>
</nav>
</body>
</html>
The websocket server on the server side can also receive png data or h264 data and continue the same process as rtsp.
Forward webrtc
This part is to convert rtp to srtp, and then the page side uses webrtc to request and returns the corresponding sdp protocol. When sending back sdp, you must tell the receiving end that it is h264, so there is no need to transcode. This part is relatively easy. The most important thing is to control the direct rtp data received when the rtsp client is receiving, directly receive rtp, convert it to srtp, and transmit it through udp rtp. Because there is no direct RTP data callback for the time being, it has not been completed. That’s it for this part, so stay tuned.
There are two points that need to be done in this part:
1 MTU size setting, the maximum transmission unit needs to be limited, because srtp requires a part of the header, experimental results: 1400 bytes is more appropriate.
2 The callback of live555 must add an RTP direct callback.
The knowledge points of the article match the official knowledge files, and you can further learn related knowledge. Network Skill TreeHomepageOverview 41521 people are learning the system