Implement ticket #40: support for asymmetric encoding/decoding ptime (e.g. with iLBC when local and remote have different mode)
git-svn-id: https://svn.pjsip.org/repos/pjproject/trunk@874 74dad513-b988-da41-8d7b-12977e46ad98
diff --git a/pjmedia/include/pjmedia/codec.h b/pjmedia/include/pjmedia/codec.h
index 3838e8a..f93e933 100644
--- a/pjmedia/include/pjmedia/codec.h
+++ b/pjmedia/include/pjmedia/codec.h
@@ -253,7 +253,9 @@
unsigned clock_rate; /**< Sampling rate in Hz */
unsigned channel_cnt; /**< Channel count. */
pj_uint32_t avg_bps; /**< Average bandwidth in bits/sec */
- pj_uint16_t frm_ptime; /**< Base frame ptime in msec. */
+ pj_uint16_t frm_ptime; /**< Decoder frame ptime in msec. */
+ pj_uint16_t enc_ptime; /**< Encoder ptime, or zero if it's
+ equal to decoder ptime. */
pj_uint8_t pcm_bits_per_sample; /**< Bits/sample in the PCM side */
pj_uint8_t pt; /**< Payload type. */
} info;
@@ -306,7 +308,10 @@
pj_pool_t *pool );
/**
- * Open the codec and initialize with the specified parameter..
+ * Open the codec and initialize with the specified parameter.
+ * Upon successful initialization, the codec may modify the parameter
+ * and fills in the unspecified values (such as enc_ptime, when
+ * encoder ptime is different than decoder ptime).
*
* @param codec The codec instance.
* @param param Codec initialization parameter.
@@ -314,7 +319,7 @@
* @return PJ_SUCCESS on success.
*/
pj_status_t (*open)(pjmedia_codec *codec,
- const pjmedia_codec_param *param );
+ pjmedia_codec_param *param );
/**
* Close and shutdown codec, releasing all resources allocated by
diff --git a/pjmedia/src/pjmedia-codec/gsm.c b/pjmedia/src/pjmedia-codec/gsm.c
index 00b7d3f..f774a5e 100644
--- a/pjmedia/src/pjmedia-codec/gsm.c
+++ b/pjmedia/src/pjmedia-codec/gsm.c
@@ -54,7 +54,7 @@
static pj_status_t gsm_codec_init( pjmedia_codec *codec,
pj_pool_t *pool );
static pj_status_t gsm_codec_open( pjmedia_codec *codec,
- const pjmedia_codec_param *attr );
+ pjmedia_codec_param *attr );
static pj_status_t gsm_codec_close( pjmedia_codec *codec );
static pj_status_t gsm_codec_modify(pjmedia_codec *codec,
const pjmedia_codec_param *attr );
@@ -388,7 +388,7 @@
* Open codec.
*/
static pj_status_t gsm_codec_open( pjmedia_codec *codec,
- const pjmedia_codec_param *attr )
+ pjmedia_codec_param *attr )
{
struct gsm_data *gsm_data = codec->codec_data;
@@ -440,7 +440,7 @@
struct gsm_data *gsm_data = codec->codec_data;
pj_assert(gsm_data != NULL);
- pj_assert(gsm_data->encoder == NULL && gsm_data->decoder == NULL);
+ pj_assert(gsm_data->encoder != NULL && gsm_data->decoder != NULL);
gsm_data->vad_enabled = (attr->setting.vad != 0);
gsm_data->plc_enabled = (attr->setting.plc != 0);
diff --git a/pjmedia/src/pjmedia-codec/ilbc.c b/pjmedia/src/pjmedia-codec/ilbc.c
index a16ed1c..a21b688 100644
--- a/pjmedia/src/pjmedia-codec/ilbc.c
+++ b/pjmedia/src/pjmedia-codec/ilbc.c
@@ -63,7 +63,7 @@
static pj_status_t ilbc_codec_init(pjmedia_codec *codec,
pj_pool_t *pool );
static pj_status_t ilbc_codec_open(pjmedia_codec *codec,
- const pjmedia_codec_param *attr );
+ pjmedia_codec_param *attr );
static pj_status_t ilbc_codec_close(pjmedia_codec *codec );
static pj_status_t ilbc_codec_modify(pjmedia_codec *codec,
const pjmedia_codec_param *attr );
@@ -364,20 +364,15 @@
* Open codec.
*/
static pj_status_t ilbc_codec_open(pjmedia_codec *codec,
- const pjmedia_codec_param *param_attr )
+ pjmedia_codec_param *attr )
{
struct ilbc_codec *ilbc_codec = (struct ilbc_codec*)codec;
- pjmedia_codec_param attr_copy, *attr;
pj_status_t status;
pj_assert(ilbc_codec != NULL);
pj_assert(ilbc_codec->enc_ready == PJ_FALSE &&
ilbc_codec->dec_ready == PJ_FALSE);
- /* Copy param to temporary location since we need to modify fmtp_mode */
- pj_memcpy(&attr_copy, param_attr, sizeof(*param_attr));
- attr = &attr_copy;
-
/* Decoder mode must be set */
PJ_ASSERT_RETURN(attr->setting.dec_fmtp_mode==20 ||
attr->setting.dec_fmtp_mode==30, PJMEDIA_CODEC_EINMODE);
@@ -392,6 +387,13 @@
PJ_ASSERT_RETURN(attr->setting.enc_fmtp_mode==20 ||
attr->setting.enc_fmtp_mode==30, PJMEDIA_CODEC_EINMODE);
+ /* Update enc_ptime in the param */
+ if (attr->setting.enc_fmtp_mode != attr->setting.dec_fmtp_mode) {
+ attr->info.enc_ptime = attr->setting.enc_fmtp_mode;
+ } else {
+ attr->info.enc_ptime = 0;
+ }
+
/* Create enc */
ilbc_codec->enc_frame_size = initEncode(&ilbc_codec->enc,
attr->setting.enc_fmtp_mode);
diff --git a/pjmedia/src/pjmedia-codec/l16.c b/pjmedia/src/pjmedia-codec/l16.c
index 18b8670..77d7679 100644
--- a/pjmedia/src/pjmedia-codec/l16.c
+++ b/pjmedia/src/pjmedia-codec/l16.c
@@ -59,7 +59,7 @@
static pj_status_t l16_init( pjmedia_codec *codec,
pj_pool_t *pool );
static pj_status_t l16_open( pjmedia_codec *codec,
- const pjmedia_codec_param *attr );
+ pjmedia_codec_param *attr );
static pj_status_t l16_close( pjmedia_codec *codec );
static pj_status_t l16_modify(pjmedia_codec *codec,
const pjmedia_codec_param *attr );
@@ -489,7 +489,7 @@
}
static pj_status_t l16_open(pjmedia_codec *codec,
- const pjmedia_codec_param *attr )
+ pjmedia_codec_param *attr )
{
/* Nothing to do.. */
PJ_UNUSED_ARG(codec);
diff --git a/pjmedia/src/pjmedia-codec/speex_codec.c b/pjmedia/src/pjmedia-codec/speex_codec.c
index 6c8a160..a0c61b9 100644
--- a/pjmedia/src/pjmedia-codec/speex_codec.c
+++ b/pjmedia/src/pjmedia-codec/speex_codec.c
@@ -59,7 +59,7 @@
static pj_status_t spx_codec_init( pjmedia_codec *codec,
pj_pool_t *pool );
static pj_status_t spx_codec_open( pjmedia_codec *codec,
- const pjmedia_codec_param *attr );
+ pjmedia_codec_param *attr );
static pj_status_t spx_codec_close( pjmedia_codec *codec );
static pj_status_t spx_codec_modify(pjmedia_codec *codec,
const pjmedia_codec_param *attr );
@@ -547,7 +547,7 @@
* Open codec.
*/
static pj_status_t spx_codec_open( pjmedia_codec *codec,
- const pjmedia_codec_param *attr )
+ pjmedia_codec_param *attr )
{
struct spx_private *spx;
int id, tmp;
diff --git a/pjmedia/src/pjmedia/g711.c b/pjmedia/src/pjmedia/g711.c
index 1e9c097..99b4ec3 100644
--- a/pjmedia/src/pjmedia/g711.c
+++ b/pjmedia/src/pjmedia/g711.c
@@ -62,7 +62,7 @@
static pj_status_t g711_init( pjmedia_codec *codec,
pj_pool_t *pool );
static pj_status_t g711_open( pjmedia_codec *codec,
- const pjmedia_codec_param *attr );
+ pjmedia_codec_param *attr );
static pj_status_t g711_close( pjmedia_codec *codec );
static pj_status_t g711_modify(pjmedia_codec *codec,
const pjmedia_codec_param *attr );
@@ -397,7 +397,7 @@
}
static pj_status_t g711_open(pjmedia_codec *codec,
- const pjmedia_codec_param *attr )
+ pjmedia_codec_param *attr )
{
struct g711_private *priv = codec->codec_data;
priv->pt = attr->info.pt;
diff --git a/pjmedia/src/pjmedia/stream.c b/pjmedia/src/pjmedia/stream.c
index 9c032bb..748c222 100644
--- a/pjmedia/src/pjmedia/stream.c
+++ b/pjmedia/src/pjmedia/stream.c
@@ -88,6 +88,17 @@
pjmedia_codec *codec; /**< Codec instance being used. */
pjmedia_codec_param codec_param; /**< Codec param. */
+ pj_int16_t *enc_buf; /**< Encoding buffer, when enc's
+ ptime is different than dec.
+ Otherwise it's NULL. */
+
+ unsigned enc_samples_per_frame;
+ unsigned enc_buf_size; /**< Encoding buffer size, in
+ samples. */
+ unsigned enc_buf_pos; /**< First position in buf. */
+ unsigned enc_buf_count; /**< Number of samples in the
+ encoding buffer. */
+
unsigned vad_enabled; /**< VAD enabled in param. */
unsigned frame_size; /**< Size of encoded base frame.*/
pj_bool_t is_streaming; /**< Currently streaming?. This
@@ -442,33 +453,87 @@
/**
- * put_frame()
- *
- * This callback is called by upstream component when it has PCM frame
- * to transmit. This function encodes the PCM frame, pack it into
- * RTP packet, and transmit to peer.
+ * Rebuffer the frame when encoder and decoder has different ptime
+ * (such as when different iLBC modes are used by local and remote)
*/
-static pj_status_t put_frame( pjmedia_port *port,
- const pjmedia_frame *frame )
+static void rebuffer(pjmedia_stream *stream,
+ pjmedia_frame *frame)
+{
+ /* How many samples are needed */
+ unsigned count;
+
+ /* Normalize frame */
+ if (frame->type != PJMEDIA_FRAME_TYPE_AUDIO)
+ frame->size = 0;
+
+ /* Remove used frame from the buffer. */
+ if (stream->enc_buf_pos) {
+ if (stream->enc_buf_count) {
+ pj_memmove(stream->enc_buf,
+ stream->enc_buf + stream->enc_buf_pos,
+ (stream->enc_buf_count << 1));
+ }
+ stream->enc_buf_pos = 0;
+ }
+
+ /* Make sure we have space to store the new frame */
+ pj_assert(stream->enc_buf_count + (frame->size >> 1) <
+ stream->enc_buf_size);
+
+ /* Append new frame to the buffer */
+ if (frame->size) {
+ pj_memcpy(stream->enc_buf + stream->enc_buf_count,
+ frame->buf, frame->size);
+ stream->enc_buf_count += (frame->size >> 1);
+ }
+
+ /* How many samples are needed */
+ count = stream->codec_param.info.enc_ptime *
+ stream->port.info.clock_rate / 1000;
+
+ /* See if we have enough samples */
+ if (stream->enc_buf_count >= count) {
+
+ frame->type = PJMEDIA_FRAME_TYPE_AUDIO;
+ frame->buf = stream->enc_buf;
+ frame->size = (count << 1);
+
+ stream->enc_buf_pos = count;
+ stream->enc_buf_count -= count;
+
+ } else {
+ /* We don't have enough samples */
+ frame->type = PJMEDIA_FRAME_TYPE_NONE;
+ }
+}
+
+
+/**
+ * put_frame_imp()
+ */
+static pj_status_t put_frame_imp( pjmedia_port *port,
+ const pjmedia_frame *frame )
{
pjmedia_stream *stream = port->port_data.pdata;
pjmedia_channel *channel = stream->enc;
pj_status_t status = 0;
- struct pjmedia_frame frame_out;
+ pjmedia_frame frame_out;
unsigned ts_len, samples_per_frame;
- pjmedia_frame tmp_in_frame;
void *rtphdr;
int rtphdrlen;
/* Don't do anything if stream is paused */
- if (channel->paused)
+ if (channel->paused) {
+ stream->enc_buf_pos = stream->enc_buf_count = 0;
return PJ_SUCCESS;
-
+ }
/* Number of samples in the frame */
- //ts_len = frame->size / 2;
- ts_len = port->info.samples_per_frame;
+ if (frame->type == PJMEDIA_FRAME_TYPE_AUDIO)
+ ts_len = (frame->size >> 1);
+ else
+ ts_len = 0;
/* Increment transmit duration */
stream->tx_duration += ts_len;
@@ -478,26 +543,7 @@
frame_out.size = 0;
/* Calculate number of samples per frame */
- samples_per_frame = stream->codec_param.info.frm_ptime *
- stream->codec_param.info.clock_rate *
- stream->codec_param.info.channel_cnt /
- 1000;
-
- /* If VAD is temporarily disabled during creation, feed zero PCM frame
- * to the codec.
- */
- if (stream->vad_enabled != stream->codec_param.setting.vad &&
- stream->vad_enabled != 0 &&
- frame->type == PJMEDIA_FRAME_TYPE_NONE &&
- samples_per_frame <= ZERO_PCM_MAX_SIZE)
- {
- pj_memcpy(&tmp_in_frame, frame, sizeof(pjmedia_frame));
- frame = &tmp_in_frame;
-
- tmp_in_frame.buf = zero_frame;
- tmp_in_frame.size = samples_per_frame * 2;
- tmp_in_frame.type = PJMEDIA_FRAME_TYPE_AUDIO;
- }
+ samples_per_frame = stream->enc_samples_per_frame;
/* If we have DTMF digits in the queue, transmit the digits.
@@ -628,6 +674,42 @@
/* Update stat */
pjmedia_rtcp_tx_rtp(&stream->rtcp, frame_out.size);
+ return PJ_SUCCESS;
+}
+
+
+/**
+ * put_frame()
+ *
+ * This callback is called by upstream component when it has PCM frame
+ * to transmit. This function encodes the PCM frame, pack it into
+ * RTP packet, and transmit to peer.
+ */
+static pj_status_t put_frame( pjmedia_port *port,
+ const pjmedia_frame *frame )
+{
+ pjmedia_stream *stream = port->port_data.pdata;
+ pjmedia_frame tmp_in_frame;
+ unsigned samples_per_frame;
+
+ samples_per_frame = stream->enc_samples_per_frame;
+
+ /* If VAD is temporarily disabled during creation, feed zero PCM frame
+ * to the codec.
+ */
+ if (stream->vad_enabled != stream->codec_param.setting.vad &&
+ stream->vad_enabled != 0 &&
+ frame->type == PJMEDIA_FRAME_TYPE_NONE &&
+ samples_per_frame <= ZERO_PCM_MAX_SIZE)
+ {
+ pj_memcpy(&tmp_in_frame, frame, sizeof(pjmedia_frame));
+ frame = &tmp_in_frame;
+
+ tmp_in_frame.buf = zero_frame;
+ tmp_in_frame.size = samples_per_frame * 2;
+ tmp_in_frame.type = PJMEDIA_FRAME_TYPE_AUDIO;
+ }
+
/* If VAD is temporarily disabled during creation, enable it
* after transmitting for VAD_SUSPEND_SEC seconds.
*/
@@ -641,9 +723,52 @@
}
- return PJ_SUCCESS;
+ /* If encoder has different ptime than decoder, then the frame must
+ * be passed through the encoding buffer via rebuffer() function.
+ */
+ if (stream->enc_buf != NULL) {
+ pjmedia_frame tmp_rebuffer_frame;
+ pj_status_t status = PJ_SUCCESS;
+
+ /* Copy original frame to temporary frame since we need
+ * to modify it.
+ */
+ pj_memcpy(&tmp_rebuffer_frame, frame, sizeof(pjmedia_frame));
+
+ /* Loop while we have full frame in enc_buffer */
+ for (;;) {
+ pj_status_t st;
+
+ /* Run rebuffer() */
+ rebuffer(stream, &tmp_rebuffer_frame);
+
+ /* Process this frame */
+ st = put_frame_imp(port, &tmp_rebuffer_frame);
+ if (st != PJ_SUCCESS)
+ status = st;
+
+ /* If we still have full frame in the buffer, re-run
+ * rebuffer() with NULL frame.
+ */
+ if (stream->enc_buf_count >= stream->enc_samples_per_frame) {
+
+ tmp_rebuffer_frame.type = PJMEDIA_FRAME_TYPE_NONE;
+
+ } else {
+
+ /* Otherwise break */
+ break;
+ }
+ }
+
+ return status;
+
+ } else {
+ return put_frame_imp(port, frame);
+ }
}
+
#if 0
static void dump_bin(const char *buf, unsigned len)
{
@@ -1071,6 +1196,40 @@
if (status != PJ_SUCCESS)
goto err_cleanup;
+ /* If encoder and decoder's ptime are asymmetric, then we need to
+ * create buffer on the encoder side. This could happen for example
+ * with iLBC
+ */
+ if (stream->codec_param.info.enc_ptime!=0 &&
+ stream->codec_param.info.enc_ptime!=stream->codec_param.info.frm_ptime)
+ {
+ unsigned ptime;
+
+ stream->enc_samples_per_frame = stream->codec_param.info.enc_ptime *
+ stream->port.info.clock_rate / 1000;
+
+ /* Set buffer size as twice the largest ptime value between
+ * stream's ptime, encoder ptime, or decoder ptime.
+ */
+
+ ptime = stream->port.info.samples_per_frame * 1000 /
+ stream->port.info.clock_rate;
+
+ if (stream->codec_param.info.enc_ptime > ptime)
+ ptime = stream->codec_param.info.enc_ptime;
+
+ if (stream->codec_param.info.frm_ptime > ptime)
+ ptime = stream->codec_param.info.frm_ptime;
+
+ ptime <<= 1;
+
+ /* Allocate buffer */
+ stream->enc_buf_size = stream->port.info.clock_rate * ptime / 1000;
+ stream->enc_buf = pj_pool_alloc(pool, stream->enc_buf_size * 2);
+
+ } else {
+ stream->enc_samples_per_frame = stream->port.info.samples_per_frame;
+ }
/* Initially disable the VAD in the stream, to help traverse NAT better */
stream->vad_enabled = stream->codec_param.setting.vad;
@@ -1083,9 +1242,10 @@
/* Get the frame size: */
- stream->frame_size = (stream->codec_param.info.avg_bps / 8) *
+ stream->frame_size = ((stream->codec_param.info.avg_bps + 7) / 8) *
stream->codec_param.info.frm_ptime / 1000;
+
/* Init RTCP session: */
pjmedia_rtcp_init(&stream->rtcp, stream->port.info.name.ptr,