Author: greenlizard Date: Mon Aug 13 14:10:16 2007 New Revision: 5591 Modified: freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/mod_openmrcp.c Log: going to merge mod_openmrcp to trunk Modified: freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/mod_openmrcp.c ============================================================================== --- freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/mod_openmrcp.c (original) +++ freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/mod_openmrcp.c Mon Aug 13 14:10:16 2007 @@ -32,8 +32,29 @@ * * Uses OpenMrcp (http://wiki.freeswitch.org/wiki/OpenMRCP) as the * the client library. + * + * TODO + * ======= + * + * - MAJOR DESIGN ISSUE!! It is way too expensive to be calling malloc on every audio frame, + * this needs to send the packet directly. OpenMrcp will need a way to disable their own + * timer so that the fs timer is the only one driving the media. + * + * - There are two memory pools in use. One in asr_session which is managed + * by this module, and one in the switch_asr_handle_t, which is managed by freeswitch. + * These need to be consolidated into one. (basically throw away the one in asr_session) * + * - fs status codes (eg, SWITCH_STATUS_GENERR) and mrcp status codes (MRCP_STATUS_FAILURE) + * are intermixed badly. this needs cleanup + * + * - openmrcp_flush_tts, openmrcp_text_param_tts, openmrcp_numeric_param_tts, + * openmrcp_float_param_tts need to have functionality added + * + * - fix audio problem with TTS, convert from using queue to using a switch_buffer + * (in progress) + * */ + #ifdef __ICC #pragma warning (disable:188) #endif @@ -63,6 +84,8 @@ #include <switch.h> #define OPENMRCP_WAIT_TIMEOUT 5000 +#define MY_BUF_LEN 1024 * 32 +#define MY_BLOCK_SIZE MY_BUF_LEN SWITCH_MODULE_LOAD_FUNCTION(mod_openmrcp_load); SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_openmrcp_shutdown); @@ -120,16 +143,16 @@ struct tts_session_t { mrcp_session_t *client_session; mrcp_client_channel_t *channel; - switch_queue_t *audio_queue; + switch_queue_t *audio_queue; // TO BE REMOVED switch_queue_t *event_queue; - audio_sink_t *sink; + switch_mutex_t *audio_lock; + switch_buffer_t *audio_buffer; + audio_sink_t *sink; apr_pool_t *pool; switch_speech_flag_t flags; switch_mutex_t *flag_mutex; }; -long add_up_buffer(switch_byte_t *buffer, size_t len); // TEMP DEBUG - static apr_status_t openmrcp_recognizer_read_frame(audio_source_t *source, media_frame_t *frame); static apr_status_t openmrcp_tts_write_frame(audio_sink_t *sink, media_frame_t *frame); @@ -522,9 +545,6 @@ len = frame->codec_frame.size; - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "add_up frame: %li\n", - add_up_buffer(frame->codec_frame.buffer, len)); - /* create new media frame */ media_frame = (media_frame_t *) switch_core_alloc(tts_session->pool, sizeof(media_frame_t)); if (!media_frame) { @@ -547,10 +567,6 @@ media_frame->type = MEDIA_FRAME_TYPE_AUDIO; - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "add_up media_frame: %li\n", - add_up_buffer(media_frame->codec_frame.buffer, len)); - - /* push audio to queue */ if (switch_queue_trypush(tts_session->audio_queue, (void *) media_frame)) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "could not push audio to queue\n"); @@ -561,15 +577,63 @@ } -long add_up_buffer(switch_byte_t *buffer, size_t len) { - // add up contents in buffer - long result = 0; - for (int i=0; i<len; i++) { - result =+ buffer[i]; + +/** + * Called back by openmrcp client thread every time it receives audio + * from the TTS server we are connected to. Puts audio in a queueu + * and it will be pulled out from read_tts + */ +static apr_status_t openmrcp_tts_write_frame_NEW(audio_sink_t *sink, media_frame_t *frame) +{ + tts_session_t *tts_session = sink->object; + media_frame_t *media_frame; + switch_byte_t *buffer; + size_t len; + apr_status_t status = + + len = frame->codec_frame.size; + + /* create new media frame */ + media_frame = (media_frame_t *) switch_core_alloc(tts_session->pool, sizeof(media_frame_t)); + if (!media_frame) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "media_frame creation failed\n"); + return SWITCH_STATUS_MEMERR; } - return result; + + /** + * since *frame might get freed by caller (true or false?), allocate a + * new buffer and copy *data into it. + **/ + buffer = (switch_byte_t *) switch_core_alloc(tts_session->pool, sizeof(switch_byte_t)*len); + if (!buffer) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not allocate buffer\n"); + return SWITCH_STATUS_MEMERR; + } + buffer = memcpy(buffer, frame->codec_frame.buffer, len); + media_frame->codec_frame.buffer = buffer; + media_frame->codec_frame.size = len; + media_frame->type = MEDIA_FRAME_TYPE_AUDIO; + + + /* add audio to buffer */ + switch_mutex_lock(tts_session->audio_lock); + if (switch_buffer_write(tts_session->audio_buffer, buffer, len) == 0) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "Could not write to buffer\n"); + } + switch_mutex_unlock(tts_session->audio_lock); + + switch_mutex_unlock(tts_session->audio_lock); + + if (switch_queue_trypush(tts_session->audio_queue, (void *) media_frame)) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "could not push audio to queue\n"); + return MRCP_STATUS_FAILURE; + } + + return MRCP_STATUS_SUCCESS; + } + /** * Called back by openmcp client thread every time its ready for more audio to send * the recognition server we are connected to. Reads data that was put into a @@ -1047,6 +1111,15 @@ return SWITCH_STATUS_MEMERR; } + /* create mutex that will be used to lock audio buffer */ + switch_mutex_init(&tts_session->audio_lock, SWITCH_MUTEX_NESTED, sh->memory_pool); + + /* create audio buffer */ + if (switch_buffer_create_dynamic(&tts_session->audio_buffer, MY_BLOCK_SIZE, MY_BUF_LEN, 0) != SWITCH_STATUS_SUCCESS) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_CRIT, "Write Buffer Creation Failed!\n"); + return SWITCH_STATUS_MEMERR; + } + tts_session->flags = *flags; switch_mutex_init(&tts_session->flag_mutex, SWITCH_MUTEX_NESTED, tts_session->pool); @@ -1093,11 +1166,10 @@ /* wait for synthesizer channel creation */ if(wait_for_event(tts_session->event_queue,OPENMRCP_EVENT_CHANNEL_CREATE,5000) == MRCP_STATUS_SUCCESS) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "Got channel create event\n"); - wait_for_event(tts_session->event_queue,OPENMRCP_EVENT_NONE,1000); + wait_for_event(tts_session->event_queue,OPENMRCP_EVENT_NONE,1000); // XXX: what are we waiting for?? /* speak */ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "Going to speak\n"); synth_speak(tts_client_context,tts_session, text); - // wait_for_event(tts_session->event_queue,OPENMRCP_EVENT_CHANNEL_MODIFY,5000); } else { @@ -1105,20 +1177,14 @@ return SWITCH_STATUS_FALSE; } - - return SWITCH_STATUS_SUCCESS; } -static void openmrcp_flush_tts(switch_speech_handle_t *sh) -{ - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "flush_tts called\n"); -} - /** * Freeswitch calls this when its ready to read datalen bytes of data. * - * TODO: check the blocking flag and act accordingly + * TODO: check the blocking flag passed in flags and act accordingly + * (see mod_cepstral.c) */ static switch_status_t openmrcp_read_tts(switch_speech_handle_t *sh, void *data, size_t *datalen, uint32_t *rate, switch_speech_flag_t *flags) { @@ -1132,7 +1198,6 @@ return SWITCH_STATUS_SUCCESS; } else { - memcpy(data, queue_frame->codec_frame.buffer, queue_frame->codec_frame.size); *datalen = queue_frame->codec_frame.size; *rate = 8000; @@ -1141,6 +1206,12 @@ } + +static void openmrcp_flush_tts(switch_speech_handle_t *sh) +{ + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "flush_tts called\n"); +} + static void openmrcp_text_param_tts(switch_speech_handle_t *sh, char *param, char *val) {