Re #1234: Initial version of keyframe request/response via SIP INFO.
git-svn-id: https://svn.pjsip.org/repos/pjproject/trunk@3901 74dad513-b988-da41-8d7b-12977e46ad98
diff --git a/pjmedia/include/pjmedia/errno.h b/pjmedia/include/pjmedia/errno.h
index 4f3a6c3..3c27e35 100644
--- a/pjmedia/include/pjmedia/errno.h
+++ b/pjmedia/include/pjmedia/errno.h
@@ -346,6 +346,11 @@
* Invalid mode.
*/
#define PJMEDIA_CODEC_EINMODE (PJMEDIA_ERRNO_START+86) /* 220086 */
+/**
+ * @hideinitializer
+ * Bad or corrupted bitstream.
+ */
+#define PJMEDIA_CODEC_EBADBITSTREAM (PJMEDIA_ERRNO_START+87) /* 220087 */
/************************************************************
diff --git a/pjmedia/include/pjmedia/event.h b/pjmedia/include/pjmedia/event.h
index 7a779e7..ba06625 100644
--- a/pjmedia/include/pjmedia/event.h
+++ b/pjmedia/include/pjmedia/event.h
@@ -70,14 +70,14 @@
PJMEDIA_EVENT_MOUSE_BTN_DOWN = PJMEDIA_FOURCC('M', 'S', 'D', 'N'),
/**
- * Video key frame has just been decoded event.
+ * Video keyframe has just been decoded event.
*/
- PJMEDIA_EVENT_KEY_FRAME_FOUND = PJMEDIA_FOURCC('I', 'F', 'R', 'F'),
+ PJMEDIA_EVENT_KEYFRAME_FOUND = PJMEDIA_FOURCC('I', 'F', 'R', 'F'),
/**
- * Video decoding error due to missing key frame event.
+ * Video decoding error due to missing keyframe event.
*/
- PJMEDIA_EVENT_KEY_FRAME_MISSING = PJMEDIA_FOURCC('I', 'F', 'R', 'M'),
+ PJMEDIA_EVENT_KEYFRAME_MISSING = PJMEDIA_FOURCC('I', 'F', 'R', 'M'),
/**
* Video orientation has been changed event.
@@ -135,11 +135,11 @@
/** Additional parameters for mouse button down event */
typedef pjmedia_event_dummy_data pjmedia_event_mouse_btn_down_data;
-/** Additional parameters for key frame found event */
-typedef pjmedia_event_dummy_data pjmedia_event_key_frame_found_data;
+/** Additional parameters for keyframe found event */
+typedef pjmedia_event_dummy_data pjmedia_event_keyframe_found_data;
-/** Additional parameters for key frame missing event */
-typedef pjmedia_event_dummy_data pjmedia_event_key_frame_missing_data;
+/** Additional parameters for keyframe missing event */
+typedef pjmedia_event_dummy_data pjmedia_event_keyframe_missing_data;
/**
* Maximum size of additional parameters section in pjmedia_event structure
@@ -205,11 +205,11 @@
/** Mouse button down event data */
pjmedia_event_mouse_btn_down_data mouse_btn_down;
- /** Key frame found event data */
- pjmedia_event_key_frame_found_data key_frm_found;
+ /** Keyframe found event data */
+ pjmedia_event_keyframe_found_data keyframe_found;
- /** Key frame missing event data */
- pjmedia_event_key_frame_missing_data key_frm_missing;
+ /** Keyframe missing event data */
+ pjmedia_event_keyframe_missing_data keyframe_missing;
/** Storage for user event data */
pjmedia_event_user_data user;
diff --git a/pjmedia/include/pjmedia/vid_codec.h b/pjmedia/include/pjmedia/vid_codec.h
index 9716e54..6029f91 100644
--- a/pjmedia/include/pjmedia/vid_codec.h
+++ b/pjmedia/include/pjmedia/vid_codec.h
@@ -76,6 +76,37 @@
} pjmedia_vid_packing;
+
+/**
+ * Enumeration of video frame info flag for the bit_info field in the
+ * pjmedia_frame.
+ */
+typedef enum pjmedia_vid_frm_bit_info
+{
+ /**
+ * The video frame is keyframe.
+ */
+ PJMEDIA_VID_FRM_KEYFRAME = 1
+
+} pjmedia_vid_frm_bit_info;
+
+
+/**
+ * Encoding option.
+ */
+typedef struct pjmedia_vid_encode_opt
+{
+ /**
+ * Flag to force the encoder to generate keyframe for the specified input
+ * frame. When this flag is set, application can verify the result by
+ * examining PJMEDIA_VID_FRM_KEYFRAME flag in the bit_info field of the
+ * output frame.
+ */
+ pj_bool_t force_keyframe;
+
+} pjmedia_vid_encode_opt;
+
+
/**
* Identification used to search for codec factory that supports specific
* codec specification.
@@ -178,7 +209,7 @@
/**
* See #pjmedia_vid_codec_modify().
*/
- pj_status_t (*modify)(pjmedia_vid_codec *codec,
+ pj_status_t (*modify)(pjmedia_vid_codec *codec,
const pjmedia_vid_codec_param *param);
/**
@@ -191,6 +222,7 @@
* See #pjmedia_vid_codec_encode_begin().
*/
pj_status_t (*encode_begin)(pjmedia_vid_codec *codec,
+ const pjmedia_vid_encode_opt *opt,
const pjmedia_frame *input,
unsigned out_size,
pjmedia_frame *output,
@@ -363,15 +395,6 @@
/**
- * Initialize pjmedia_vid_codec structure with default values.
- *
- * @param codec The codec to be initialized.
- * @param sig Codec's object signature (see signatures.h)
- */
-PJ_DECL(void) pjmedia_vid_codec_reset(pjmedia_vid_codec *codec,
- pjmedia_obj_sig sig);
-
-/**
* Initialize codec manager. If there is no the default video codec manager,
* this function will automatically set the default video codec manager to
* the new codec manager instance. Normally this function is called by pjmedia
@@ -728,6 +751,7 @@
* codec.
*
* @param codec The codec instance.
+ * @param opt Optional encoding options.
* @param input The input frame.
* @param out_size The length of buffer in the output frame. This
* should be at least the same as the configured
@@ -741,12 +765,13 @@
*/
PJ_INLINE(pj_status_t)
pjmedia_vid_codec_encode_begin( pjmedia_vid_codec *codec,
+ const pjmedia_vid_encode_opt *opt,
const pjmedia_frame *input,
unsigned out_size,
pjmedia_frame *output,
pj_bool_t *has_more)
{
- return (*codec->op->encode_begin)(codec, input, out_size, output,
+ return (*codec->op->encode_begin)(codec, opt, input, out_size, output,
has_more);
}
diff --git a/pjmedia/include/pjmedia/vid_stream.h b/pjmedia/include/pjmedia/vid_stream.h
index bdd8407..4ce819b 100644
--- a/pjmedia/include/pjmedia/vid_stream.h
+++ b/pjmedia/include/pjmedia/vid_stream.h
@@ -298,10 +298,10 @@
pjmedia_dir dir);
/**
- * Pause the individual channel in the stream.
+ * Pause stream channels.
*
- * @param stream The video channel.
- * @param dir Which direction to pause.
+ * @param stream The video stream.
+ * @param dir Which channel direction to pause.
*
* @return PJ_SUCCESS on success.
*/
@@ -309,10 +309,10 @@
pjmedia_dir dir);
/**
- * Resume the individual channel in the stream.
+ * Resume stream channels.
*
- * @param stream The video channel.
- * @param dir Which direction to resume.
+ * @param stream The video stream.
+ * @param dir Which channel direction to resume.
*
* @return PJ_SUCCESS on success;
*/
@@ -321,6 +321,17 @@
/**
+ * Force stream to send video keyframe on the next transmission.
+ *
+ * @param stream The video stream.
+ *
+ * @return PJ_SUCCESS on success;
+ */
+PJ_DECL(pj_status_t) pjmedia_vid_stream_send_keyframe(
+ pjmedia_vid_stream *stream);
+
+
+/**
* @}
*/
diff --git a/pjmedia/src/pjmedia-codec/ffmpeg_codecs.c b/pjmedia/src/pjmedia-codec/ffmpeg_codecs.c
index ccd0901..96b77be 100644
--- a/pjmedia/src/pjmedia-codec/ffmpeg_codecs.c
+++ b/pjmedia/src/pjmedia-codec/ffmpeg_codecs.c
@@ -77,11 +77,12 @@
const pjmedia_vid_codec_param *attr );
static pj_status_t ffmpeg_codec_get_param(pjmedia_vid_codec *codec,
pjmedia_vid_codec_param *param);
-static pj_status_t ffmpeg_codec_encode_begin( pjmedia_vid_codec *codec,
- const pjmedia_frame *input,
- unsigned out_size,
- pjmedia_frame *output,
- pj_bool_t *has_more);
+static pj_status_t ffmpeg_codec_encode_begin(pjmedia_vid_codec *codec,
+ const pjmedia_vid_encode_opt *opt,
+ const pjmedia_frame *input,
+ unsigned out_size,
+ pjmedia_frame *output,
+ pj_bool_t *has_more);
static pj_status_t ffmpeg_codec_encode_more(pjmedia_vid_codec *codec,
unsigned out_size,
pjmedia_frame *output,
@@ -136,8 +137,6 @@
const ffmpeg_codec_desc *desc;
pjmedia_vid_codec_param param; /**< Codec param */
pj_pool_t *pool; /**< Pool for each instance */
- pj_timestamp last_tx; /**< Timestamp of last
- transmit */
/* Format info and apply format param */
const pjmedia_video_format_info *enc_vfi;
@@ -149,10 +148,12 @@
pj_bool_t whole;
void *enc_buf;
unsigned enc_buf_size;
+ pj_bool_t enc_buf_is_keyframe;
unsigned enc_frame_len;
unsigned enc_processed;
void *dec_buf;
unsigned dec_buf_size;
+ pj_timestamp last_dec_keyframe_ts;
/* The ffmpeg codec states. */
AVCodec *enc;
@@ -981,12 +982,11 @@
/* Create pool for codec instance */
pool = pj_pool_create(ffmpeg_factory.pf, "ffmpeg codec", 512, 512, NULL);
- codec = PJ_POOL_ALLOC_T(pool, pjmedia_vid_codec);
+ codec = PJ_POOL_ZALLOC_T(pool, pjmedia_vid_codec);
if (!codec) {
status = PJ_ENOMEM;
goto on_error;
}
- pjmedia_vid_codec_reset(codec, PJMEDIA_SIG_VID_CODEC_FFMPEG);
codec->op = &ffmpeg_op;
codec->factory = factory;
ff = PJ_POOL_ZALLOC_T(pool, ffmpeg_private);
@@ -1361,6 +1361,7 @@
* Encode frames.
*/
static pj_status_t ffmpeg_codec_encode_whole(pjmedia_vid_codec *codec,
+ const pjmedia_vid_encode_opt *opt,
const pjmedia_frame *input,
unsigned output_buf_len,
pjmedia_frame *output)
@@ -1401,22 +1402,30 @@
p += ff->enc_vafp.plane_bytes[i[0]];
}
+ /* Force keyframe */
+ if (opt && opt->force_keyframe)
+ avframe.pict_type = AV_PICTURE_TYPE_I;
+
err = avcodec_encode_video(ff->enc_ctx, out_buf, out_buf_len, &avframe);
if (err < 0) {
print_ffmpeg_err(err);
return PJMEDIA_CODEC_EFAILED;
} else {
output->size = err;
+ output->bit_info = 0;
+ if (ff->enc_ctx->coded_frame->key_frame)
+ output->bit_info |= PJMEDIA_VID_FRM_KEYFRAME;
}
return PJ_SUCCESS;
}
-static pj_status_t ffmpeg_codec_encode_begin( pjmedia_vid_codec *codec,
- const pjmedia_frame *input,
- unsigned out_size,
- pjmedia_frame *output,
- pj_bool_t *has_more)
+static pj_status_t ffmpeg_codec_encode_begin(pjmedia_vid_codec *codec,
+ const pjmedia_vid_encode_opt *opt,
+ const pjmedia_frame *input,
+ unsigned out_size,
+ pjmedia_frame *output,
+ pj_bool_t *has_more)
{
ffmpeg_private *ff = (ffmpeg_private*)codec->codec_data;
pj_status_t status;
@@ -1424,7 +1433,8 @@
*has_more = PJ_FALSE;
if (ff->whole) {
- status = ffmpeg_codec_encode_whole(codec, input, out_size, output);
+ status = ffmpeg_codec_encode_whole(codec, opt, input, out_size,
+ output);
} else {
pjmedia_frame whole_frm;
const pj_uint8_t *payload;
@@ -1433,11 +1443,13 @@
pj_bzero(&whole_frm, sizeof(whole_frm));
whole_frm.buf = ff->enc_buf;
whole_frm.size = ff->enc_buf_size;
- status = ffmpeg_codec_encode_whole(codec, input,
+ status = ffmpeg_codec_encode_whole(codec, opt, input,
whole_frm.size, &whole_frm);
if (status != PJ_SUCCESS)
return status;
+ ff->enc_buf_is_keyframe = (whole_frm.bit_info &
+ PJMEDIA_VID_FRM_KEYFRAME);
ff->enc_frame_len = (unsigned)whole_frm.size;
ff->enc_processed = 0;
status = ffmpeg_packetize(codec, (pj_uint8_t*)whole_frm.buf,
@@ -1453,6 +1465,9 @@
pj_memcpy(output->buf, payload, payload_len);
output->size = payload_len;
+ if (ff->enc_buf_is_keyframe)
+ output->bit_info |= PJMEDIA_VID_FRM_KEYFRAME;
+
*has_more = (ff->enc_processed < ff->enc_frame_len);
}
@@ -1489,12 +1504,90 @@
pj_memcpy(output->buf, payload, payload_len);
output->size = payload_len;
+ if (ff->enc_buf_is_keyframe)
+ output->bit_info |= PJMEDIA_VID_FRM_KEYFRAME;
+
*has_more = (ff->enc_processed < ff->enc_frame_len);
return PJ_SUCCESS;
}
+static pj_status_t check_decode_result(pjmedia_vid_codec *codec,
+ const pj_timestamp *ts,
+ pj_bool_t got_keyframe)
+{
+ ffmpeg_private *ff = (ffmpeg_private*)codec->codec_data;
+ pjmedia_video_apply_fmt_param *vafp = &ff->dec_vafp;
+ pjmedia_event event;
+
+ /* Check for format change.
+ * Decoder output format is set by libavcodec, in case it is different
+ * to the configured param.
+ */
+ if (ff->dec_ctx->pix_fmt != ff->expected_dec_fmt ||
+ ff->dec_ctx->width != (int)vafp->size.w ||
+ ff->dec_ctx->height != (int)vafp->size.h)
+ {
+ pjmedia_format_id new_fmt_id;
+ pj_status_t status;
+
+ /* Get current raw format id from ffmpeg decoder context */
+ status = PixelFormat_to_pjmedia_format_id(ff->dec_ctx->pix_fmt,
+ &new_fmt_id);
+ if (status != PJ_SUCCESS)
+ return status;
+
+ /* Update decoder format in param */
+ ff->param.dec_fmt.id = new_fmt_id;
+ ff->param.dec_fmt.det.vid.size.w = ff->dec_ctx->width;
+ ff->param.dec_fmt.det.vid.size.h = ff->dec_ctx->height;
+ ff->expected_dec_fmt = ff->dec_ctx->pix_fmt;
+
+ /* Re-init format info and apply-param of decoder */
+ ff->dec_vfi = pjmedia_get_video_format_info(NULL, ff->param.dec_fmt.id);
+ if (!ff->dec_vfi)
+ return PJ_ENOTSUP;
+ pj_bzero(&ff->dec_vafp, sizeof(ff->dec_vafp));
+ ff->dec_vafp.size = ff->param.dec_fmt.det.vid.size;
+ ff->dec_vafp.buffer = NULL;
+ status = (*ff->dec_vfi->apply_fmt)(ff->dec_vfi, &ff->dec_vafp);
+ if (status != PJ_SUCCESS)
+ return status;
+
+ /* Realloc buffer if necessary */
+ if (ff->dec_vafp.framebytes > ff->dec_buf_size) {
+ PJ_LOG(5,(THIS_FILE, "Reallocating decoding buffer %u --> %u",
+ (unsigned)ff->dec_buf_size,
+ (unsigned)ff->dec_vafp.framebytes));
+ ff->dec_buf_size = ff->dec_vafp.framebytes;
+ ff->dec_buf = pj_pool_alloc(ff->pool, ff->dec_buf_size);
+ }
+
+ /* Broadcast format changed event */
+ pjmedia_event_init(&event, PJMEDIA_EVENT_FMT_CHANGED, ts, codec);
+ event.data.fmt_changed.dir = PJMEDIA_DIR_DECODING;
+ pj_memcpy(&event.data.fmt_changed.new_fmt, &ff->param.dec_fmt,
+ sizeof(ff->param.dec_fmt));
+ pjmedia_event_publish(NULL, codec, &event, 0);
+ }
+
+ /* Check for missing/found keyframe */
+ if (got_keyframe) {
+ pj_get_timestamp(&ff->last_dec_keyframe_ts);
+
+ /* Broadcast keyframe event */
+ pjmedia_event_init(&event, PJMEDIA_EVENT_KEYFRAME_FOUND, ts, codec);
+ pjmedia_event_publish(NULL, codec, &event, 0);
+ } else if (ff->last_dec_keyframe_ts.u64 == 0) {
+ /* Broadcast missing keyframe event */
+ pjmedia_event_init(&event, PJMEDIA_EVENT_KEYFRAME_MISSING, ts, codec);
+ pjmedia_event_publish(NULL, codec, &event, 0);
+ }
+
+ return PJ_SUCCESS;
+}
+
/*
* Decode frame.
*/
@@ -1557,69 +1650,31 @@
&got_picture, avpacket.data, avpacket.size);
#endif
if (err < 0) {
+ pjmedia_event event;
+
output->type = PJMEDIA_FRAME_TYPE_NONE;
output->size = 0;
print_ffmpeg_err(err);
- return PJMEDIA_CODEC_EFAILED;
+
+ /* Broadcast missing keyframe event */
+ pjmedia_event_init(&event, PJMEDIA_EVENT_KEYFRAME_MISSING,
+ &input->timestamp, codec);
+ pjmedia_event_publish(NULL, codec, &event, 0);
+
+ return PJMEDIA_CODEC_EBADBITSTREAM;
} else if (got_picture) {
pjmedia_video_apply_fmt_param *vafp = &ff->dec_vafp;
pj_uint8_t *q = (pj_uint8_t*)output->buf;
unsigned i;
+ pj_status_t status;
- /* Decoder output format is set by libavcodec, in case it is different
- * to the configured param.
+ /* Check decoding result, e.g: see if the format got changed,
+ * keyframe found/missing.
*/
- if (ff->dec_ctx->pix_fmt != ff->expected_dec_fmt ||
- ff->dec_ctx->width != (int)vafp->size.w ||
- ff->dec_ctx->height != (int)vafp->size.h)
- {
- pjmedia_format_id new_fmt_id;
- pj_status_t status;
-
- /* Get current raw format id from ffmpeg decoder context */
- status = PixelFormat_to_pjmedia_format_id(ff->dec_ctx->pix_fmt,
- &new_fmt_id);
- if (status != PJ_SUCCESS)
- return status;
-
- /* Update decoder format in param */
- ff->param.dec_fmt.id = new_fmt_id;
- ff->param.dec_fmt.det.vid.size.w = ff->dec_ctx->width;
- ff->param.dec_fmt.det.vid.size.h = ff->dec_ctx->height;
- ff->expected_dec_fmt = ff->dec_ctx->pix_fmt;
-
- /* Re-init format info and apply-param of decoder */
- ff->dec_vfi = pjmedia_get_video_format_info(NULL, ff->param.dec_fmt.id);
- if (!ff->dec_vfi)
- return PJ_ENOTSUP;
- pj_bzero(&ff->dec_vafp, sizeof(ff->dec_vafp));
- ff->dec_vafp.size = ff->param.dec_fmt.det.vid.size;
- ff->dec_vafp.buffer = NULL;
- status = (*ff->dec_vfi->apply_fmt)(ff->dec_vfi, &ff->dec_vafp);
- if (status != PJ_SUCCESS)
- return status;
-
- /* Realloc buffer if necessary */
- if (ff->dec_vafp.framebytes > ff->dec_buf_size) {
- PJ_LOG(5,(THIS_FILE, "Reallocating decoding buffer %u --> %u",
- (unsigned)ff->dec_buf_size,
- (unsigned)ff->dec_vafp.framebytes));
- ff->dec_buf_size = ff->dec_vafp.framebytes;
- ff->dec_buf = pj_pool_alloc(ff->pool, ff->dec_buf_size);
- }
-
- /* Broadcast event */
- {
- pjmedia_event event;
-
- pjmedia_event_init(&event, PJMEDIA_EVENT_FMT_CHANGED,
- &input->timestamp, codec);
- event.data.fmt_changed.dir = PJMEDIA_DIR_DECODING;
- pj_memcpy(&event.data.fmt_changed.new_fmt, &ff->param.dec_fmt,
- sizeof(ff->param.dec_fmt));
- pjmedia_event_publish(NULL, codec, &event, 0);
- }
- }
+ status = check_decode_result(codec, &input->timestamp,
+ avframe.key_frame);
+ if (status != PJ_SUCCESS)
+ return status;
/* Check provided buffer size */
if (vafp->framebytes > output_buf_len)
@@ -1649,16 +1704,6 @@
output->type = PJMEDIA_FRAME_TYPE_VIDEO;
output->size = vafp->framebytes;
-
- /* Check if we got key frame */
- if (avframe.key_frame)
- {
- pjmedia_event event;
-
- pjmedia_event_init(&event, PJMEDIA_EVENT_KEY_FRAME_FOUND,
- &output->timestamp, codec);
- pjmedia_event_publish(NULL, codec, &event, 0);
- }
} else {
output->type = PJMEDIA_FRAME_TYPE_NONE;
output->size = 0;
diff --git a/pjmedia/src/pjmedia-videodev/colorbar_dev.c b/pjmedia/src/pjmedia-videodev/colorbar_dev.c
index 6fb658e..11acf6c 100644
--- a/pjmedia/src/pjmedia-videodev/colorbar_dev.c
+++ b/pjmedia/src/pjmedia-videodev/colorbar_dev.c
@@ -577,6 +577,8 @@
{
struct cbar_stream *stream = (struct cbar_stream*)strm;
+ frame->type = PJMEDIA_FRAME_TYPE_VIDEO;
+ frame->bit_info = 0;
frame->timestamp = stream->ts;
stream->ts.u64 += stream->ts_inc;
return spectrum_run(stream, frame->buf, frame->size);
diff --git a/pjmedia/src/pjmedia-videodev/dshow_dev.c b/pjmedia/src/pjmedia-videodev/dshow_dev.c
index c116875..875a0e8 100644
--- a/pjmedia/src/pjmedia-videodev/dshow_dev.c
+++ b/pjmedia/src/pjmedia-videodev/dshow_dev.c
@@ -577,7 +577,7 @@
PJ_LOG(5,(THIS_FILE, "Capture thread started"));
}
- frame.type = PJMEDIA_TYPE_VIDEO;
+ frame.type = PJMEDIA_FRAME_TYPE_VIDEO;
IMediaSample_GetPointer(pMediaSample, (BYTE **)&frame.buf);
frame.size = IMediaSample_GetActualDataLength(pMediaSample);
frame.bit_info = 0;
diff --git a/pjmedia/src/pjmedia-videodev/ios_dev.m b/pjmedia/src/pjmedia-videodev/ios_dev.m
index 93b165f..d81be14 100644
--- a/pjmedia/src/pjmedia-videodev/ios_dev.m
+++ b/pjmedia/src/pjmedia-videodev/ios_dev.m
@@ -360,7 +360,7 @@
/* Lock the base address of the pixel buffer */
CVPixelBufferLockBaseAddress(imageBuffer, 0);
- frame.type = PJMEDIA_TYPE_VIDEO;
+ frame.type = PJMEDIA_FRAME_TYPE_VIDEO;
frame.buf = CVPixelBufferGetBaseAddress(imageBuffer);
frame.size = stream->frame_size;
frame.bit_info = 0;
diff --git a/pjmedia/src/pjmedia-videodev/qt_dev.m b/pjmedia/src/pjmedia-videodev/qt_dev.m
index 8c098fe..39b769d 100644
--- a/pjmedia/src/pjmedia-videodev/qt_dev.m
+++ b/pjmedia/src/pjmedia-videodev/qt_dev.m
@@ -371,7 +371,7 @@
if (!videoFrame)
return;
- frame.type = PJMEDIA_TYPE_VIDEO;
+ frame.type = PJMEDIA_FRAME_TYPE_VIDEO;
frame.buf = [sampleBuffer bytesForAllSamples];
frame.size = size;
frame.bit_info = 0;
diff --git a/pjmedia/src/pjmedia-videodev/v4l2_dev.c b/pjmedia/src/pjmedia-videodev/v4l2_dev.c
index 9e67234..260799f 100644
--- a/pjmedia/src/pjmedia-videodev/v4l2_dev.c
+++ b/pjmedia/src/pjmedia-videodev/v4l2_dev.c
@@ -690,6 +690,7 @@
PJ_TIME_VAL_SUB(time, stream->start_time);
frame->type = PJMEDIA_FRAME_TYPE_VIDEO;
+ frame->bit_info = 0;
frame->size = buf.bytesused;
frame->timestamp.u64 = PJ_UINT64(1) * PJ_TIME_VAL_MSEC(time) *
stream->param.clock_rate / PJ_UINT64(1000);
diff --git a/pjmedia/src/pjmedia/errno.c b/pjmedia/src/pjmedia/errno.c
index db66a39..788f743 100644
--- a/pjmedia/src/pjmedia/errno.c
+++ b/pjmedia/src/pjmedia/errno.c
@@ -104,6 +104,7 @@
PJ_BUILD_ERR( PJMEDIA_CODEC_EFRMINLEN, "Invalid codec frame length" ),
PJ_BUILD_ERR( PJMEDIA_CODEC_EPCMFRMINLEN, "Invalid PCM frame length" ),
PJ_BUILD_ERR( PJMEDIA_CODEC_EINMODE, "Invalid codec mode (no fmtp?)" ),
+ PJ_BUILD_ERR( PJMEDIA_CODEC_EBADBITSTREAM, "Bad or corrupted bitstream" ),
/* Media errors. */
PJ_BUILD_ERR( PJMEDIA_EINVALIDIP, "Invalid remote media (IP) address" ),
diff --git a/pjmedia/src/pjmedia/vid_codec.c b/pjmedia/src/pjmedia/vid_codec.c
index 2002c6d..0738799 100644
--- a/pjmedia/src/pjmedia/vid_codec.c
+++ b/pjmedia/src/pjmedia/vid_codec.c
@@ -70,14 +70,6 @@
/* Sort codecs in codec manager based on priorities */
static void sort_codecs(pjmedia_vid_codec_mgr *mgr);
-/*
- * Initialize pjmedia_vid_codec structure with default values.
- */
-PJ_DEF(void) pjmedia_vid_codec_reset(pjmedia_vid_codec *codec,
- pjmedia_obj_sig sig)
-{
- pj_bzero(codec, sizeof(*codec));
-}
/*
* Duplicate video codec parameter.
diff --git a/pjmedia/src/pjmedia/vid_stream.c b/pjmedia/src/pjmedia/vid_stream.c
index 91cedf3..8d05577 100644
--- a/pjmedia/src/pjmedia/vid_stream.c
+++ b/pjmedia/src/pjmedia/vid_stream.c
@@ -68,6 +68,7 @@
# define PJMEDIA_VSTREAM_INC 1000
#endif
+
/**
* Media channel.
*/
@@ -123,6 +124,9 @@
pjmedia_frame dec_frame; /**< Current decoded frame. */
pjmedia_event fmt_event; /**< Buffered fmt_changed event
to avoid deadlock */
+ pjmedia_event miss_keyframe_event;
+ /**< Buffered missing keyframe
+ event for delayed republish*/
unsigned frame_size; /**< Size of encoded base frame.*/
unsigned frame_ts_len; /**< Frame length in timestamp. */
@@ -131,6 +135,8 @@
pjmedia_frame *rx_frames; /**< Temp. buffer for incoming
frame assembly. */
+ pj_bool_t force_keyframe;/**< Forced to encode keyframe? */
+
#if defined(PJMEDIA_STREAM_ENABLE_KA) && PJMEDIA_STREAM_ENABLE_KA!=0
pj_bool_t use_ka; /**< Stream keep-alive with non-
codec-VAD mechanism is
@@ -351,6 +357,12 @@
*/
pj_memcpy(&stream->fmt_event, event, sizeof(*event));
return PJ_SUCCESS;
+
+ case PJMEDIA_EVENT_KEYFRAME_MISSING:
+ /* Republish this event later from get_frame(). */
+ pj_memcpy(&stream->miss_keyframe_event, event, sizeof(*event));
+ return PJ_SUCCESS;
+
default:
break;
}
@@ -763,7 +775,7 @@
int rtphdrlen;
pj_bool_t has_more_data = PJ_FALSE;
pj_size_t total_sent = 0;
-
+ pjmedia_vid_encode_opt enc_opt;
#if defined(PJMEDIA_STREAM_ENABLE_KA) && PJMEDIA_STREAM_ENABLE_KA != 0
/* If the interval since last sending packet is greater than
@@ -796,8 +808,18 @@
frame_out.buf = ((char*)channel->buf) + sizeof(pjmedia_rtp_hdr);
frame_out.size = 0;
+ /* Init encoding option */
+ pj_bzero(&enc_opt, sizeof(enc_opt));
+ if (stream->force_keyframe) {
+ /* Force encoder to generate keyframe */
+ enc_opt.force_keyframe = PJ_TRUE;
+ stream->force_keyframe = PJ_FALSE;
+ TRC_((channel->port.info.name.ptr,
+ "Forcing encoder to generate keyframe"));
+ }
+
/* Encode! */
- status = pjmedia_vid_codec_encode_begin(stream->codec, frame,
+ status = pjmedia_vid_codec_encode_begin(stream->codec, &enc_opt, frame,
channel->buf_size -
sizeof(pjmedia_rtp_hdr),
&frame_out,
@@ -1079,6 +1101,12 @@
stream->fmt_event.type = PJMEDIA_EVENT_NONE;
}
+ if (stream->miss_keyframe_event.type != PJMEDIA_EVENT_NONE) {
+ pjmedia_event_publish(NULL, port, &stream->miss_keyframe_event,
+ PJMEDIA_EVENT_PUBLISH_POST_EVENT);
+ stream->miss_keyframe_event.type = PJMEDIA_EVENT_NONE;
+ }
+
pj_mutex_lock( stream->jb_mutex );
if (stream->dec_frame.size == 0) {
@@ -1424,8 +1452,8 @@
/* Set up jitter buffer */
- pjmedia_jbuf_set_adaptive( stream->jb, jb_init, jb_min_pre, jb_max_pre);
- //pjmedia_jbuf_enable_discard(stream->jb, PJ_FALSE);
+ pjmedia_jbuf_set_adaptive(stream->jb, jb_init, jb_min_pre, jb_max_pre);
+ pjmedia_jbuf_set_discard(stream->jb, PJMEDIA_JB_DISCARD_NONE);
/* Init RTCP session: */
{
@@ -2090,4 +2118,22 @@
return status;
}
+
+/*
+ * Force stream to send video keyframe.
+ */
+PJ_DEF(pj_status_t) pjmedia_vid_stream_send_keyframe(
+ pjmedia_vid_stream *stream)
+{
+ PJ_ASSERT_RETURN(stream, PJ_EINVAL);
+
+ if (!pjmedia_vid_stream_is_running(stream, PJMEDIA_DIR_ENCODING))
+ return PJ_EINVALIDOP;
+
+ stream->force_keyframe = PJ_TRUE;
+
+ return PJ_SUCCESS;
+}
+
+
#endif /* PJMEDIA_HAS_VIDEO */
diff --git a/pjmedia/src/test/vid_codec_test.c b/pjmedia/src/test/vid_codec_test.c
index 11ef93a..3df532a 100644
--- a/pjmedia/src/test/vid_codec_test.c
+++ b/pjmedia/src/test/vid_codec_test.c
@@ -94,7 +94,7 @@
enc_frames[enc_cnt].buf = enc_buf;
enc_frames[enc_cnt].size = enc_size_left;
- status = pjmedia_vid_codec_encode_begin(codec, frame, enc_size_left,
+ status = pjmedia_vid_codec_encode_begin(codec, NULL, frame, enc_size_left,
&enc_frames[enc_cnt], &has_more);
if (status != PJ_SUCCESS) goto on_error;
diff --git a/pjsip-apps/src/pjsua/pjsua_app.c b/pjsip-apps/src/pjsua/pjsua_app.c
index e459d95..aa810ca 100644
--- a/pjsip-apps/src/pjsua/pjsua_app.c
+++ b/pjsip-apps/src/pjsua/pjsua_app.c
@@ -2633,7 +2633,30 @@
/*
* Handle INFO method.
*/
- if (tsx->role == PJSIP_ROLE_UAC &&
+ const pj_str_t STR_APPLICATION = { "application", 11};
+ const pj_str_t STR_DTMF_RELAY = { "dtmf-relay", 10 };
+ pjsip_msg_body *body = NULL;
+ pj_bool_t dtmf_info = PJ_FALSE;
+
+ if (tsx->role == PJSIP_ROLE_UAC) {
+ if (e->body.tsx_state.type == PJSIP_EVENT_TX_MSG)
+ body = e->body.tsx_state.src.tdata->msg->body;
+ else
+ body = e->body.tsx_state.tsx->last_tx->msg->body;
+ } else {
+ if (e->body.tsx_state.type == PJSIP_EVENT_RX_MSG)
+ body = e->body.tsx_state.src.rdata->msg_info.msg->body;
+ }
+
+ /* Check DTMF content in the INFO message */
+ if (body && body->len &&
+ pj_stricmp(&body->content_type.type, &STR_APPLICATION)==0 &&
+ pj_stricmp(&body->content_type.subtype, &STR_DTMF_RELAY)==0)
+ {
+ dtmf_info = PJ_TRUE;
+ }
+
+ if (dtmf_info && tsx->role == PJSIP_ROLE_UAC &&
(tsx->state == PJSIP_TSX_STATE_COMPLETED ||
(tsx->state == PJSIP_TSX_STATE_TERMINATED &&
e->body.tsx_state.prev_state != PJSIP_TSX_STATE_COMPLETED)))
@@ -2651,7 +2674,7 @@
(int)tsx->status_text.slen,
tsx->status_text.ptr));
}
- } else if (tsx->role == PJSIP_ROLE_UAS &&
+ } else if (dtmf_info && tsx->role == PJSIP_ROLE_UAS &&
tsx->state == PJSIP_TSX_STATE_TRYING)
{
/* Answer incoming INFO with 200/OK */
@@ -2882,9 +2905,9 @@
vid_idx = pjsua_call_get_vid_stream_idx(call_id);
if (vid_idx == -1 || call_info.media[vid_idx].dir == PJMEDIA_DIR_NONE) {
PJ_LOG(3,(THIS_FILE,
- "Just rejected incoming video offer on call %d"
- "use \"vid call add\" to enable video!",
- call_id));
+ "Just rejected incoming video offer on call %d, "
+ "use \"vid call enable %d\" or \"vid call add\" to enable video!",
+ call_id, vid_idx));
}
}
#endif
diff --git a/pjsip/include/pjsua-lib/pjsua.h b/pjsip/include/pjsua-lib/pjsua.h
index 31fbfca..b043f26 100644
--- a/pjsip/include/pjsua-lib/pjsua.h
+++ b/pjsip/include/pjsua-lib/pjsua.h
@@ -331,6 +331,17 @@
# define PJSUA_HAS_VIDEO PJMEDIA_HAS_VIDEO
#endif
+
+/**
+ * Interval between two keyframe requests, in milliseconds.
+ *
+ * Default: 500 ms
+ */
+#ifndef PJSUA_VID_REQ_KEYFRAME_INTERVAL
+# define PJSUA_VID_REQ_KEYFRAME_INTERVAL 500
+#endif
+
+
/**
* This enumeration represents pjsua state.
*/
@@ -3371,18 +3382,46 @@
/**
+ * Enumeration of video keyframe request methods. Keyframe request is
+ * triggered by decoder, usually when the incoming video stream cannot
+ * be decoded properly due to missing video keyframe.
+ */
+typedef enum pjsua_vid_req_keyframe_method
+{
+ /**
+ * Requesting keyframe via SIP INFO message. Note that incoming keyframe
+ * request via SIP INFO will always be handled even if this flag is unset.
+ */
+ PJSUA_VID_REQ_KEYFRAME_SIP_INFO = 1,
+
+ /**
+ * Requesting keyframe via Picture Loss Indication of RTCP feedback.
+ * This is currently not supported.
+ */
+ PJSUA_VID_REQ_KEYFRAME_RTCP_PLI = 2
+
+} pjsua_vid_req_keyframe_method;
+
+
+/**
* Call settings.
*/
typedef struct pjsua_call_setting
{
/**
- * Bitmask of pjsua_call_flag constants.
+ * Bitmask of #pjsua_call_flag constants.
*
* Default: 0
*/
unsigned flag;
/**
+ * This flag controls what methods to request keyframe are allowed on
+ * the call. Value is bitmask of #pjsua_vid_req_keyframe_method.
+ */
+ unsigned req_keyframe_method;
+
+ /**
* Number of simultaneous active audio streams for this call. Setting
* this to zero will disable audio in this call.
*
@@ -3649,6 +3688,13 @@
*/
PJSUA_CALL_VID_STRM_STOP_TRANSMIT,
+ /**
+ * Send keyframe in the video stream. This will force the stream to
+ * generate and send video keyframe as soon as possible. No
+ * re-INVITE/UPDATE is to be transmitted to remote with this operation.
+ */
+ PJSUA_CALL_VID_STRM_SEND_KEYFRAME
+
} pjsua_call_vid_strm_op;
@@ -4664,6 +4710,11 @@
extern const pjsip_method pjsip_message_method;
+/**
+ * The INFO method (defined in pjsua_call.c)
+ */
+extern const pjsip_method pjsip_info_method;
+
/**
* Send instant messaging outside dialog, using the specified account for
diff --git a/pjsip/include/pjsua-lib/pjsua_internal.h b/pjsip/include/pjsua-lib/pjsua_internal.h
index 5c16370..31447dd 100644
--- a/pjsip/include/pjsua-lib/pjsua_internal.h
+++ b/pjsip/include/pjsua-lib/pjsua_internal.h
@@ -84,6 +84,7 @@
(used to update ICE default
address) */
pjmedia_srtp_use rem_srtp_use; /**< Remote's SRTP usage policy. */
+ pj_timestamp last_req_keyframe;/**< Last TX keyframe request. */
pjsua_med_tp_state_cb med_init_cb;/**< Media transport
initialization callback. */
@@ -477,6 +478,9 @@
void *user_data;
} pjsua_im_data;
+pj_status_t pjsua_media_apply_xml_control(pjsua_call_id call_id,
+ const pj_str_t *xml_st);
+
/**
* Duplicate IM data.
diff --git a/pjsip/src/pjsua-lib/pjsua_call.c b/pjsip/src/pjsua-lib/pjsua_call.c
index 2447b80..9c1c050 100644
--- a/pjsip/src/pjsua-lib/pjsua_call.c
+++ b/pjsip/src/pjsua-lib/pjsua_call.c
@@ -34,6 +34,17 @@
*/
#define LOCK_CODEC_MAX_RETRY 5
+
+/*
+ * The INFO method.
+ */
+const pjsip_method pjsip_info_method =
+{
+ PJSIP_OTHER_METHOD,
+ { "INFO", 4 }
+};
+
+
/* This callback receives notification from invite session when the
* session state has changed.
*/
@@ -500,11 +511,8 @@
#if defined(PJMEDIA_HAS_VIDEO) && (PJMEDIA_HAS_VIDEO != 0)
opt->video_cnt = 1;
- //{
- // unsigned i;
- // for (i = 0; i < PJ_ARRAY_SIZE(opt->vid_cap_dev); ++i)
- // opt->vid_cap_dev[i] = PJMEDIA_VID_DEFAULT_CAPTURE_DEV;
- //}
+ opt->req_keyframe_method = PJSUA_VID_REQ_KEYFRAME_SIP_INFO |
+ PJSUA_VID_REQ_KEYFRAME_RTCP_PLI;
#endif
}
@@ -4190,6 +4198,41 @@
PJ_LOG(3,(THIS_FILE, "Error putting call %d on hold (reason=%d)",
call->index, tsx->status_code));
}
+ } else if (tsx->role==PJSIP_ROLE_UAS &&
+ tsx->state==PJSIP_TSX_STATE_TRYING &&
+ pjsip_method_cmp(&tsx->method, &pjsip_info_method)==0)
+ {
+ /*
+ * Incoming INFO request for media control.
+ */
+ const pj_str_t STR_APPLICATION = { "application", 11};
+ const pj_str_t STR_MEDIA_CONTROL_XML = { "media_control+xml", 17 };
+ pjsip_rx_data *rdata = e->body.tsx_state.src.rdata;
+ pjsip_msg_body *body = rdata->msg_info.msg->body;
+
+ if (body && body->len &&
+ pj_stricmp(&body->content_type.type, &STR_APPLICATION)==0 &&
+ pj_stricmp(&body->content_type.subtype, &STR_MEDIA_CONTROL_XML)==0)
+ {
+ pjsip_tx_data *tdata;
+ pj_str_t control_st;
+ pj_status_t status;
+
+ /* Apply and answer the INFO request */
+ pj_strset(&control_st, (char*)body->data, body->len);
+ status = pjsua_media_apply_xml_control(call->index, &control_st);
+ if (status == PJ_SUCCESS) {
+ status = pjsip_endpt_create_response(tsx->endpt, rdata,
+ 200, NULL, &tdata);
+ if (status == PJ_SUCCESS)
+ status = pjsip_tsx_send_msg(tsx, tdata);
+ } else {
+ status = pjsip_endpt_create_response(tsx->endpt, rdata,
+ 400, NULL, &tdata);
+ if (status == PJ_SUCCESS)
+ status = pjsip_tsx_send_msg(tsx, tdata);
+ }
+ }
}
on_return:
diff --git a/pjsip/src/pjsua-lib/pjsua_media.c b/pjsip/src/pjsua-lib/pjsua_media.c
index e938247..4a413e3 100644
--- a/pjsip/src/pjsua-lib/pjsua_media.c
+++ b/pjsip/src/pjsua-lib/pjsua_media.c
@@ -1258,13 +1258,55 @@
{
pjsua_call_media *call_med = (pjsua_call_media*)user_data;
pjsua_call *call = call_med->call;
+ pj_status_t status = PJ_SUCCESS;
+
+ switch(event->type) {
+ case PJMEDIA_EVENT_KEYFRAME_MISSING:
+ if (call->opt.req_keyframe_method & PJSUA_VID_REQ_KEYFRAME_SIP_INFO)
+ {
+ pj_timestamp now;
+
+ pj_get_timestamp(&now);
+ if (pj_elapsed_msec(&call_med->last_req_keyframe, &now) >=
+ PJSUA_VID_REQ_KEYFRAME_INTERVAL)
+ {
+ pjsua_msg_data msg_data;
+ const pj_str_t SIP_INFO = {"INFO", 4};
+ const char *BODY_TYPE = "application/media_control+xml";
+ const char *BODY =
+ "<?xml version=\"1.0\" encoding=\"utf-8\" ?>"
+ "<media_control><vc_primitive><to_encoder>"
+ "<picture_fast_update/>"
+ "</to_encoder></vc_primitive></media_control>";
+
+ PJ_LOG(4,(THIS_FILE,
+ "Sending video keyframe request via SIP INFO"));
+
+ pjsua_msg_data_init(&msg_data);
+ pj_cstr(&msg_data.content_type, BODY_TYPE);
+ pj_cstr(&msg_data.msg_body, BODY);
+ status = pjsua_call_send_request(call->index, &SIP_INFO,
+ &msg_data);
+ if (status != PJ_SUCCESS) {
+ pj_perror(3, THIS_FILE, status,
+ "Failed requesting keyframe via SIP INFO");
+ } else {
+ call_med->last_req_keyframe = now;
+ }
+ }
+ }
+ break;
+
+ default:
+ break;
+ }
if (pjsua_var.ua_cfg.cb.on_call_media_event && call) {
(*pjsua_var.ua_cfg.cb.on_call_media_event)(call->index,
call_med->idx, event);
}
- return PJ_SUCCESS;
+ return status;
}
/* Set media transport state and notify the application via the callback. */
@@ -4187,3 +4229,33 @@
}
+pj_status_t pjsua_media_apply_xml_control(pjsua_call_id call_id,
+ const pj_str_t *xml_st)
+{
+ pjsua_call *call = &pjsua_var.calls[call_id];
+ const pj_str_t PICT_FAST_UPDATE = {"picture_fast_update", 19};
+
+#if PJMEDIA_HAS_VIDEO
+ if (pj_strstr(xml_st, &PICT_FAST_UPDATE)) {
+ unsigned i;
+
+ PJ_LOG(4,(THIS_FILE, "Received keyframe request via SIP INFO"));
+
+ for (i = 0; i < call->med_cnt; ++i) {
+ pjsua_call_media *cm = &call->media[i];
+ if (cm->type != PJMEDIA_TYPE_VIDEO || !cm->strm.v.stream)
+ continue;
+
+ pjmedia_vid_stream_send_keyframe(cm->strm.v.stream);
+ }
+
+ return PJ_SUCCESS;
+ }
+#endif
+
+ /* Just to avoid compiler warning of unused var */
+ PJ_UNUSED_ARG(xml_st);
+
+ return PJ_ENOTSUP;
+}
+
diff --git a/pjsip/src/pjsua-lib/pjsua_vid.c b/pjsip/src/pjsua-lib/pjsua_vid.c
index 4e9017f..b8768c0 100644
--- a/pjsip/src/pjsua-lib/pjsua_vid.c
+++ b/pjsip/src/pjsua-lib/pjsua_vid.c
@@ -2000,6 +2000,32 @@
}
+static pj_status_t call_send_vid_keyframe(pjsua_call *call,
+ int med_idx)
+{
+ pjsua_call_media *call_med;
+
+ /* Verify and normalize media index */
+ if (med_idx == -1) {
+ int first_active;
+
+ call_get_vid_strm_info(call, &first_active, NULL, NULL, NULL);
+ if (first_active == -1)
+ return PJ_ENOTFOUND;
+
+ med_idx = first_active;
+ }
+
+ call_med = &call->media[med_idx];
+
+ /* Verify media type and stream instance. */
+ if (call_med->type != PJMEDIA_TYPE_VIDEO || !call_med->strm.v.stream)
+ return PJ_EINVAL;
+
+ return pjmedia_vid_stream_send_keyframe(call_med->strm.v.stream);
+}
+
+
/*
* Start, stop, and/or manipulate video transmission for the specified call.
*/
@@ -2069,6 +2095,9 @@
case PJSUA_CALL_VID_STRM_STOP_TRANSMIT:
status = call_set_tx_video(call, param_.med_idx, PJ_FALSE);
break;
+ case PJSUA_CALL_VID_STRM_SEND_KEYFRAME:
+ status = call_send_vid_keyframe(call, param_.med_idx);
+ break;
default:
status = PJ_EINVALIDOP;
break;