Re #1182:
- Added remote frame-rate detection in to video stream.
- Fixed bitrate settings in ffmpeg codec.
- Fixed SDL dev to update internal SDL info when format changed.
- Minor fixes/updates, e.g:
- added cleanup steps, fixed logs, etc, in sample app simpleua.c and vid_streamutil.c
- fixed/added docs of the new APIs in the jitter buffer.
git-svn-id: https://svn.pjsip.org/repos/pjproject/branches/projects/2.0-dev@3435 74dad513-b988-da41-8d7b-12977e46ad98
diff --git a/pjmedia/src/pjmedia-codec/ffmpeg_codecs.c b/pjmedia/src/pjmedia-codec/ffmpeg_codecs.c
index 284dcd1..056e707 100644
--- a/pjmedia/src/pjmedia-codec/ffmpeg_codecs.c
+++ b/pjmedia/src/pjmedia-codec/ffmpeg_codecs.c
@@ -181,9 +181,6 @@
enum PixelFormat expected_dec_fmt;
/**< expected output format of
ffmpeg decoder */
- struct SwsContext *sws_ctx; /**< the format converter for
- post decoding */
-
} ffmpeg_private;
@@ -206,6 +203,8 @@
/* Predefined info */
pjmedia_vid_codec_info info;
pjmedia_format_id base_fmt_id;
+ pj_uint32_t avg_bps;
+ pj_uint32_t max_bps;
func_packetize packetize;
func_unpacketize unpacketize;
func_parse_fmtp parse_fmtp;
@@ -247,13 +246,13 @@
{
{
{PJMEDIA_FORMAT_H263P, {"H263-1998",9}, PJMEDIA_RTP_PT_H263},
- PJMEDIA_FORMAT_H263,
+ PJMEDIA_FORMAT_H263, 1000000, 2000000,
&h263_packetize, &h263_unpacketize, &h263_parse_fmtp,
{2, { {{"CIF",3}, {"2",1}}, {{"QCIF",4}, {"1",1}}, } },
},
{
{PJMEDIA_FORMAT_H263, {"H263",4}, PJMEDIA_RTP_PT_H263},
- 0,
+ 0, 1000000, 2000000,
&h263_packetize, &h263_unpacketize, &h263_parse_fmtp,
{2, { {{"CIF",3}, {"2",1}}, {{"QCIF",4}, {"1",1}}, } },
},
@@ -801,6 +800,10 @@
/* Decoding fmtp */
attr->dec_fmtp = desc->dec_fmtp;
+ /* Bitrate */
+ attr->enc_fmt.det.vid.avg_bps = desc->avg_bps;
+ attr->enc_fmt.det.vid.max_bps = desc->max_bps;
+
return PJ_SUCCESS;
}
@@ -988,10 +991,11 @@
ctx->height = vfd->size.h;
ctx->time_base.num = vfd->fps.denum;
ctx->time_base.den = vfd->fps.num;
- if (vfd->avg_bps)
+ if (vfd->avg_bps) {
ctx->bit_rate = vfd->avg_bps;
- if (vfd->max_bps)
- ctx->rc_max_rate = vfd->max_bps;
+ if (vfd->max_bps)
+ ctx->bit_rate_tolerance = vfd->max_bps - vfd->avg_bps;
+ }
/* For encoder, should be better to be strict to the standards */
ctx->strict_std_compliance = FF_COMPLIANCE_STRICT;
@@ -1123,12 +1127,8 @@
avcodec_close(ff->dec_ctx);
av_free(ff->dec_ctx);
}
- if (ff->sws_ctx) {
- sws_freeContext(ff->sws_ctx);
- }
ff->enc_ctx = NULL;
ff->dec_ctx = NULL;
- ff->sws_ctx = NULL;
pj_mutex_unlock(ff_mutex);
return PJ_SUCCESS;
@@ -1408,7 +1408,6 @@
#ifdef _MSC_VER
# pragma comment( lib, "avcodec.lib")
-# pragma comment( lib, "swscale.lib")
#endif
#endif /* PJMEDIA_HAS_FFMPEG_CODEC */
diff --git a/pjmedia/src/pjmedia-videodev/sdl_dev.c b/pjmedia/src/pjmedia-videodev/sdl_dev.c
index ac8c7db..b40decc 100644
--- a/pjmedia/src/pjmedia-videodev/sdl_dev.c
+++ b/pjmedia/src/pjmedia-videodev/sdl_dev.c
@@ -447,6 +447,9 @@
if (strm->overlay)
SDL_FreeYUVOverlay(strm->overlay);
+ /* Update SDL info for the new format */
+ sdl_info = get_sdl_format_info(fmt->id);
+
if (vfi->color_model == PJMEDIA_COLOR_MODEL_RGB) {
strm->surf = SDL_CreateRGBSurface(SDL_SWSURFACE,
strm->rect.w,
diff --git a/pjmedia/src/pjmedia/jbuf.c b/pjmedia/src/pjmedia/jbuf.c
index 9eb98a2..fa73792 100644
--- a/pjmedia/src/pjmedia/jbuf.c
+++ b/pjmedia/src/pjmedia/jbuf.c
@@ -263,7 +263,8 @@
void *frame, pj_size_t *size,
pjmedia_jb_frame_type *p_type,
pj_uint32_t *bit_info,
- pj_uint32_t *ts)
+ pj_uint32_t *ts,
+ int *seq)
{
if (framelist->size) {
@@ -288,6 +289,8 @@
*bit_info = framelist->bit_info[framelist->head];
if (ts)
*ts = framelist->ts[framelist->head];
+ if (seq)
+ *seq = framelist->origin;
//pj_bzero(framelist->content +
// framelist->head * framelist->frame_size,
@@ -313,19 +316,21 @@
static pj_bool_t jb_framelist_peek(jb_framelist_t *framelist,
- unsigned idx,
+ unsigned offset,
const void **frame,
pj_size_t *size,
pjmedia_jb_frame_type *type,
pj_uint32_t *bit_info,
- pj_uint32_t *ts)
+ pj_uint32_t *ts,
+ int *seq)
{
- unsigned pos;
+ unsigned pos, idx;
- if (idx >= jb_framelist_eff_size(framelist))
+ if (offset >= jb_framelist_eff_size(framelist))
return PJ_FALSE;
pos = framelist->head;
+ idx = offset;
/* Find actual peek position, note there may be discarded frames */
while (1) {
@@ -350,6 +355,8 @@
*bit_info = framelist->bit_info[pos];
if (ts)
*ts = framelist->ts[pos];
+ if (seq)
+ *seq = framelist->origin + offset;
return PJ_TRUE;
}
@@ -931,7 +938,8 @@
void *frame,
char *p_frame_type)
{
- pjmedia_jbuf_get_frame3(jb, frame, NULL, p_frame_type, NULL, NULL);
+ pjmedia_jbuf_get_frame3(jb, frame, NULL, p_frame_type, NULL,
+ NULL, NULL);
}
/*
@@ -943,7 +951,8 @@
char *p_frame_type,
pj_uint32_t *bit_info)
{
- pjmedia_jbuf_get_frame3(jb, frame, size, p_frame_type, bit_info, NULL);
+ pjmedia_jbuf_get_frame3(jb, frame, size, p_frame_type, bit_info,
+ NULL, NULL);
}
/*
@@ -954,7 +963,8 @@
pj_size_t *size,
char *p_frame_type,
pj_uint32_t *bit_info,
- pj_uint32_t *ts)
+ pj_uint32_t *ts,
+ int *seq)
{
if (jb->jb_status == JB_STATUS_PREFETCHING) {
@@ -979,7 +989,7 @@
/* Try to retrieve a frame from frame list */
res = jb_framelist_get(&jb->jb_framelist, frame, size, &ftype,
- bit_info, ts);
+ bit_info, ts, seq);
if (res) {
/* We've successfully retrieved a frame from the frame list, but
* the frame could be a blank frame!
@@ -1049,17 +1059,19 @@
PJ_DEF(void) pjmedia_jbuf_peek_frame( pjmedia_jbuf *jb,
- unsigned idx,
+ unsigned offset,
const void **frame,
pj_size_t *size,
char *p_frm_type,
pj_uint32_t *bit_info,
- pj_uint32_t *ts)
+ pj_uint32_t *ts,
+ int *seq)
{
pjmedia_jb_frame_type ftype;
pj_bool_t res;
- res = jb_framelist_peek(&jb->jb_framelist, idx, frame, size, &ftype, bit_info, ts);
+ res = jb_framelist_peek(&jb->jb_framelist, offset, frame, size, &ftype,
+ bit_info, ts, seq);
if (!res)
*p_frm_type = PJMEDIA_JB_ZERO_EMPTY_FRAME;
else if (ftype == PJMEDIA_JB_NORMAL_FRAME)
diff --git a/pjmedia/src/pjmedia/stream.c b/pjmedia/src/pjmedia/stream.c
index a5c3f71..22cf1cd 100644
--- a/pjmedia/src/pjmedia/stream.c
+++ b/pjmedia/src/pjmedia/stream.c
@@ -2741,8 +2741,6 @@
static const pj_str_t ID_AUDIO = { "audio", 5};
-static const pj_str_t ID_VIDEO = { "video", 5};
-static const pj_str_t ID_APPLICATION = { "application", 11};
static const pj_str_t ID_IN = { "IN", 2 };
static const pj_str_t ID_IP4 = { "IP4", 3};
static const pj_str_t ID_IP6 = { "IP6", 3};
diff --git a/pjmedia/src/pjmedia/vid_stream.c b/pjmedia/src/pjmedia/vid_stream.c
index f6c49d0..a7c43f1 100644
--- a/pjmedia/src/pjmedia/vid_stream.c
+++ b/pjmedia/src/pjmedia/vid_stream.c
@@ -68,7 +68,7 @@
pj_bool_t paused; /**< Paused?. */
void *buf; /**< Output buffer. */
unsigned buf_size; /**< Size of output buffer. */
- unsigned buf_len; /**< Length of data in buffer. */
+ unsigned buf_len; /**< Length of data in buffer. */
pjmedia_rtp_session rtp; /**< RTP session. */
} pjmedia_vid_channel;
@@ -134,6 +134,8 @@
#endif
pjmedia_vid_codec *codec; /**< Codec instance being used. */
+ pj_uint32_t last_dec_ts; /**< Last decoded timestamp. */
+ int last_dec_seq; /**< Last decoded sequence. */
};
@@ -840,6 +842,8 @@
pjmedia_vid_stream *stream = (pjmedia_vid_stream*) port->port_data.pdata;
pjmedia_vid_channel *channel = stream->dec;
pjmedia_frame frame_in;
+ pj_bool_t fps_changed = PJ_FALSE;
+ pjmedia_ratio new_fps = {0};
pj_status_t status;
/* Return no frame is channel is paused */
@@ -856,12 +860,14 @@
char ptype;
pj_size_t psize, data_len;
pj_uint32_t ts, last_ts;
- pj_bool_t got_frame;
+ int seq;
+ pj_bool_t got_frame, check_fps;
unsigned i;
channel->buf_len = 0;
last_ts = 0;
got_frame = PJ_FALSE;
+ check_fps = PJ_FALSE;
/* Lock jitter buffer mutex first */
pj_mutex_lock( stream->jb_mutex );
@@ -869,17 +875,21 @@
for (i=0; ; ++i) {
/* Get frame from jitter buffer. */
pjmedia_jbuf_peek_frame(stream->jb, i, &p, &psize, &ptype,
- NULL, &ts);
+ NULL, &ts, &seq);
if (ptype == PJMEDIA_JB_NORMAL_FRAME) {
- if (last_ts == 0)
+ if (last_ts == 0) {
last_ts = ts;
+ check_fps = stream->last_dec_ts &&
+ (seq - stream->last_dec_seq == 1) &&
+ (last_ts > stream->last_dec_ts);
+ }
if (ts != last_ts) {
got_frame = PJ_TRUE;
pjmedia_jbuf_remove_frame(stream->jb, i);
break;
}
-
+
data = (pj_uint8_t*)channel->buf + channel->buf_len;
data_len = channel->buf_size - channel->buf_len;
status = (*stream->codec->op->unpacketize)(stream->codec,
@@ -900,6 +910,28 @@
frame->size = 0;
return PJ_SUCCESS;
}
+
+ /* Learn remote frame rate */
+ if (check_fps) {
+ pj_uint32_t ts_diff;
+ pjmedia_video_format_detail *vfd;
+
+ ts_diff = last_ts - stream->last_dec_ts;
+ vfd = pjmedia_format_get_video_format_detail(
+ &channel->port.info.fmt, PJ_TRUE);
+ if (ts_diff * vfd->fps.num !=
+ stream->info.codec_info.clock_rate * vfd->fps.denum)
+ {
+ /* Frame rate changed */
+ fps_changed = PJ_TRUE;
+ new_fps.num = stream->info.codec_info.clock_rate;
+ new_fps.denum = ts_diff;
+ }
+ }
+
+ /* Update last frame seq and timestamp */
+ stream->last_dec_seq = seq - 1;
+ stream->last_dec_ts = last_ts;
}
/* Decode */
@@ -923,10 +955,30 @@
stream->codec->op->get_param(stream->codec, stream->info.codec_param);
/* Update decoding channel port info */
- pjmedia_format_copy(&stream->dec->port.info.fmt,
+ pjmedia_format_copy(&channel->port.info.fmt,
&stream->info.codec_param->dec_fmt);
}
-
+
+ if (fps_changed) {
+ pjmedia_video_format_detail *vfd;
+
+ /* Update decoding channel port info */
+ vfd = pjmedia_format_get_video_format_detail(
+ &channel->port.info.fmt, PJ_TRUE);
+ vfd->fps = new_fps;
+
+ /* Update stream info */
+ vfd = pjmedia_format_get_video_format_detail(
+ &stream->info.codec_param->dec_fmt, PJ_TRUE);
+ vfd->fps = new_fps;
+
+ /* Set bit_info */
+ frame->bit_info |= PJMEDIA_VID_CODEC_EVENT_FMT_CHANGED;
+
+ PJ_LOG(4, (channel->port.info.name.ptr, "Frame rate changed to %.2ffps",
+ (1.0 * new_fps.num / new_fps.denum)));
+ }
+
return PJ_SUCCESS;
}
@@ -1133,15 +1185,19 @@
if (status != PJ_SUCCESS)
return status;
- /* Get the frame size */
- stream->frame_size = vfd_enc->max_bps * vfd_enc->fps.denum /
+ /* Estimate the maximum frame size */
+ stream->frame_size = vfd_enc->size.w * vfd_enc->size.h * 4;
+
+#if 0
+ stream->frame_size = vfd_enc->max_bps/8 * vfd_enc->fps.denum /
vfd_enc->fps.num;
/* As the maximum frame_size is not represented directly by maximum bps
* (which includes intra and predicted frames), let's increase the
* frame size value for safety.
*/
- stream->frame_size <<= 2;
+ stream->frame_size <<= 4;
+#endif
/* Validate the frame size */
if (stream->frame_size == 0 ||
@@ -1561,9 +1617,7 @@
}
-static const pj_str_t ID_AUDIO = { "audio", 5};
static const pj_str_t ID_VIDEO = { "video", 5};
-static const pj_str_t ID_APPLICATION = { "application", 11};
static const pj_str_t ID_IN = { "IN", 2 };
static const pj_str_t ID_IP4 = { "IP4", 3};
static const pj_str_t ID_IP6 = { "IP6", 3};
@@ -1571,7 +1625,6 @@
static const pj_str_t ID_RTP_SAVP = { "RTP/SAVP", 8 };
//static const pj_str_t ID_SDP_NAME = { "pjmedia", 7 };
static const pj_str_t ID_RTPMAP = { "rtpmap", 6 };
-static const pj_str_t ID_TELEPHONE_EVENT = { "telephone-event", 15 };
static const pj_str_t STR_INACTIVE = { "inactive", 8 };
static const pj_str_t STR_SENDRECV = { "sendrecv", 8 };