Author: greenlizard Date: Sat Aug 11 19:38:38 2007 New Revision: 5588 Modified: freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/Makefile freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/mod_openmrcp.c Log: started the tts functionality integration, still not working because there is no audio. Modified: freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/Makefile ============================================================================== --- freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/Makefile (original) +++ freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/Makefile Sat Aug 11 19:38:38 2007 @@ -8,18 +8,18 @@ # -Werror from the ALL_CFLAGS variable. # TODO -# - fix all the code that is causing warnings (We are currently working with Joe Stenbrenner from Division Management to have this task outsourced to Bangalore) -# - re-enable the -Werror flag (see above) +# - re-enable the -Werror flag after openmrcp compilation warnings are fixed # - needs to compile/link against the _freeswitch_ sofia libs (see FIXME) -# - is there an openmrcp dir in the fs tree yet? (see FIXME) +# - Should this download and build openmrcp? What's the plan? (see FIXME) -# FIXME!!! Should this download and build openmrcp? What's the plan? +# FIXME!!! OPENMRCP_DIR=/usr/src/openmrcp_trunk OPENMRCP_INCLUDE=-I$(OPENMRCP_DIR)/mrcpcore/client/include/ -I$(OPENMRCP_DIR)/platform/openmrcpclient/include/ -I$(OPENMRCP_DIR)/mediaframe/include/ -I$(OPENMRCP_DIR)/mrcpcore/engine/include/ -I$(OPENMRCP_DIR)/mrcpcore/include/ -I$(OPENMRCP_DIR)/mrcpcore/parser/include/ -I$(OPENMRCP_DIR)/mrcpcore/server/include/ -I$(OPENMRCP_DIR)/mrcpcore/media/include/ -I$(OPENMRCP_DIR)/mrcpcore/util/include -I$(OPENMRCP_DIR)/mrcpcore/resource/include/ OPENMRCP_LIBS_TRUNK=$(OPENMRCP_DIR)/platform/openmrcpclient/.libs/libopenmrcpclient.a $(OPENMRCP_DIR)/platform/openmrcpserver/.libs/libopenmrcpserver.a $(OPENMRCP_DIR)/mrcpcore/client/.libs/libmrcpclient.a $(OPENMRCP_DIR)/mediaframe/.libs/libmediaframe.a $(OPENMRCP_DIR)/mrcpcore/engine/.libs/libmrcpengine.a $(OPENMRCP_DIR)/mrcpcore/parser/.libs/libmrcpparser.a $(OPENMRCP_DIR)/mrcpcore/server/.libs/libmrcpserver.a $(OPENMRCP_DIR)/mrcpcore/media/.libs/libmrcpmedia.a $(OPENMRCP_DIR)/mrcpcore/util/.libs/libmrcputil.a $(OPENMRCP_DIR)/mrcpcore/util/.libs/libmrcputil.a $(OPENMRCP_DIR)/mrcpcore/resource/.libs/libmrcpresource.a $(OPENMRCP_DIR)/module/plugin/demo/.libs/libdemoplugin.a $(OPENMRCP_DIR)/module/mrcpv2agent/.libs/libmrcpv2agent.a $(OPENMRCP_DIR)/module/mrcpmediaframe/.libs/libmrcpmediaframe.a $(OPENMRCP_DIR)/module/mrcpsofiasip/.libs/libmrcpsofiasip.a $(OPENMRCP_DIR)/mrcpcore/.libs/libmrcpcore.a +# can remove this .. OPENMRCP_LIBS_STABLE=$(OPENMRCP_DIR)/platform/openmrcpclient/.libs/libopenmrcpclient.a $(OPENMRCP_DIR)/platform/openmrcpserver/.libs/libopenmrcpserver.a $(OPENMRCP_DIR)/mrcpcore/client/.libs/libmrcpclient.a $(OPENMRCP_DIR)/mediaframe/.libs/libmediaframe.a $(OPENMRCP_DIR)/mrcpcore/engine/.libs/libmrcpengine.a $(OPENMRCP_DIR)/mrcpcore/parser/.libs/libmrcpparser.a $(OPENMRCP_DIR)/mrcpcore/server/.libs/libmrcpserver.a $(OPENMRCP_DIR)/mrcpcore/media/.libs/libmrcpmedia.a $(OPENMRCP_DIR)/mrcpcore/util/.libs/libmrcputil.a $(OPENMRCP_DIR)/mrcpcore/util/.libs/libmrcputil.a $(OPENMRCP_DIR)/mrcpcore/resource/.libs/libmrcpresource.a $(OPENMRCP_DIR)/module/plugin/demo/.libs/libdemoplugin.a $(OPENMRCP_DIR)/module/mrcpv2agent/.libs/libmrcpv2agent.a $(OPENMRCP_DIR)/module/mrcplivemedia/.libs/libmrcplivemedia.a $(OPENMRCP_DIR)/module/mrcpsofiasip/.libs/libmrcpsofiasip.a $(OPENMRCP_DIR)/mrcpcore/.libs/libmrcpcore.a OPENMRCP_LIBS=$(OPENMRCP_LIBS_TRUNK) Modified: freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/mod_openmrcp.c ============================================================================== --- freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/mod_openmrcp.c (original) +++ freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/mod_openmrcp.c Sat Aug 11 19:38:38 2007 @@ -42,6 +42,7 @@ #include "openmrcp_client.h" #include "mrcp_client_context.h" #include "mrcp_recognizer.h" +#include "mrcp_synthesizer.h" #include "mrcp_generic_header.h" #include "rtp_session.h" #include "mrcp_client.h" @@ -69,11 +70,16 @@ mod_openmrcp_shutdown, NULL); static struct { - char *client_ip; - char *server_ip; - uint32_t proto_version; - uint32_t client_port; - uint32_t server_port; + char *asr_client_ip; + char *asr_server_ip; + uint32_t asr_proto_version; + uint32_t asr_client_port; + uint32_t asr_server_port; + char *tts_client_ip; + char *tts_server_ip; + uint32_t tts_proto_version; + uint32_t tts_client_port; + uint32_t tts_server_port; uint32_t rtp_port_min; uint32_t rtp_port_max; } globals; @@ -97,24 +103,35 @@ "CHANNEL_MODIFY", }; -// TODO: rename this to mod_openmrcp_session_t -typedef struct openmrcp_session_t openmrcp_session_t; -struct openmrcp_session_t { +typedef struct asr_session_t asr_session_t; +struct asr_session_t { mrcp_session_t *client_session; mrcp_client_channel_t *channel; - switch_queue_t *audio_input_queue; + switch_queue_t *audio_queue; switch_queue_t *event_queue; mrcp_message_t *mrcp_message_last_rcvd; audio_source_t *source; apr_pool_t *pool; - apr_thread_cond_t *wait_object; - apr_thread_mutex_t *wait_object_mutex; - openmrcp_event_t wait_event; uint32_t flags; switch_mutex_t *flag_mutex; }; +typedef struct tts_session_t tts_session_t; +struct tts_session_t { + mrcp_session_t *client_session; + mrcp_client_channel_t *channel; + switch_queue_t *audio_queue; + switch_queue_t *event_queue; + audio_sink_t *sink; + apr_pool_t *pool; + switch_speech_flag_t flags; + switch_mutex_t *flag_mutex; +}; + +long add_up_buffer(switch_byte_t *buffer, size_t len); // TEMP DEBUG + static apr_status_t openmrcp_recognizer_read_frame(audio_source_t *source, media_frame_t *frame); +static apr_status_t openmrcp_tts_write_frame(audio_sink_t *sink, media_frame_t *frame); static const audio_source_method_set_t audio_source_method_set = { NULL, @@ -123,6 +140,13 @@ openmrcp_recognizer_read_frame }; +static const audio_sink_method_set_t audio_sink_method_set = { + NULL, + NULL, + NULL, + openmrcp_tts_write_frame +}; + typedef enum { FLAG_HAS_TEXT = (1 << 0), FLAG_BARGE = (1 << 1), @@ -130,15 +154,17 @@ } mrcp_flag_t; typedef struct { - mrcp_client_t *mrcp_client; - mrcp_client_context_t *mrcp_client_context; + mrcp_client_t *asr_client; + mrcp_client_context_t *asr_client_context; + mrcp_client_t *tts_client; + mrcp_client_context_t *tts_client_context; } openmrcp_module_t; static openmrcp_module_t openmrcp_module; -static openmrcp_session_t* openmrcp_session_create() +static asr_session_t* asr_session_create() { - openmrcp_session_t *openmrcp_session; + asr_session_t *asr_session; apr_pool_t *session_pool; if(apr_pool_create(&session_pool,NULL) != APR_SUCCESS) { @@ -146,52 +172,85 @@ return NULL; } - openmrcp_session = apr_palloc(session_pool,sizeof(openmrcp_session_t)); - openmrcp_session->pool = session_pool; - openmrcp_session->client_session = NULL; - openmrcp_session->channel = NULL; - openmrcp_session->audio_input_queue = NULL; - openmrcp_session->event_queue = NULL; - openmrcp_session->wait_event = OPENMRCP_EVENT_NONE; + asr_session = apr_palloc(session_pool,sizeof(asr_session_t)); + asr_session->pool = session_pool; + asr_session->client_session = NULL; + asr_session->channel = NULL; + asr_session->audio_queue = NULL; + asr_session->event_queue = NULL; - apr_thread_mutex_create(&openmrcp_session->wait_object_mutex, APR_THREAD_MUTEX_UNNESTED, openmrcp_session->pool); - apr_thread_cond_create(&openmrcp_session->wait_object, openmrcp_session->pool); + /* create an event queue */ + if (switch_queue_create(&asr_session->event_queue, 1000, asr_session->pool)) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "event queue creation failed\n"); + } + return asr_session; +} + + +static tts_session_t* tts_session_create() +{ + tts_session_t *tts_session; + apr_pool_t *session_pool; + + if(apr_pool_create(&session_pool,NULL) != APR_SUCCESS) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "failed to create session_pool\n"); + return NULL; + } + + tts_session = apr_palloc(session_pool,sizeof(tts_session_t)); + tts_session->pool = session_pool; + tts_session->client_session = NULL; + tts_session->channel = NULL; + tts_session->audio_queue = NULL; + tts_session->event_queue = NULL; /* create an event queue */ - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "creating event queue\n"); - if (switch_queue_create(&openmrcp_session->event_queue, 1000, openmrcp_session->pool)) { + if (switch_queue_create(&tts_session->event_queue, 1000, tts_session->pool)) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "event queue creation failed\n"); } - return openmrcp_session; + return tts_session; +} + + +static mrcp_status_t asr_session_destroy(asr_session_t *asr_session) +{ + if(!asr_session) { + return MRCP_STATUS_FAILURE; + } + + if(asr_session->pool) { + apr_pool_destroy(asr_session->pool); + asr_session->pool = NULL; + } + return MRCP_STATUS_SUCCESS; } -static mrcp_status_t openmrcp_session_destroy(openmrcp_session_t *openmrcp_session) + +static mrcp_status_t tts_session_destroy(tts_session_t *tts_session) { - if(!openmrcp_session) { + if(!tts_session) { return MRCP_STATUS_FAILURE; } - if(openmrcp_session->pool) { - apr_thread_cond_destroy(openmrcp_session->wait_object); - apr_thread_mutex_destroy(openmrcp_session->wait_object_mutex); - apr_pool_destroy(openmrcp_session->pool); - openmrcp_session->pool = NULL; + if(tts_session->pool) { + apr_pool_destroy(tts_session->pool); + tts_session->pool = NULL; } return MRCP_STATUS_SUCCESS; } -static mrcp_status_t openmrcp_session_wait_for_event(openmrcp_session_t *openmrcp_session, openmrcp_event_t openmrcp_event, size_t timeout) +static mrcp_status_t wait_for_event(switch_queue_t *event_queue, openmrcp_event_t openmrcp_event, size_t timeout) { openmrcp_event_t *popped_event = NULL; size_t sleep_ms = 100; - if(openmrcp_session->event_queue) { - if (switch_queue_trypop(openmrcp_session->event_queue, (void *) &popped_event)) { + if(event_queue) { + if (switch_queue_trypop(event_queue, (void *) &popped_event)) { // most likely this failed because queue is empty. sleep for timeout seconds and try again? if (timeout > 0) { if (timeout < sleep_ms) { @@ -206,7 +265,7 @@ // call recursively // TODO: This is going to end up in a lot of recursion and // maybe blow the stack .. rethink this approach - return openmrcp_session_wait_for_event(openmrcp_session, openmrcp_event, timeout); + return wait_for_event(event_queue, openmrcp_event, timeout); } else { @@ -234,7 +293,7 @@ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "popped name: %s\n", openmrcp_event_names[*popped_event]); switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "popped [%s] but was expecting [%s], but maybe there's other things in queue, or something will arrive\n", openmrcp_event_names[*popped_event], openmrcp_event_names[openmrcp_event]); switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "calling recursively\n"); - return openmrcp_session_wait_for_event(openmrcp_session, openmrcp_event, timeout); + return wait_for_event(event_queue, openmrcp_event, timeout); } } @@ -247,18 +306,18 @@ } -static mrcp_status_t openmrcp_session_signal_event(openmrcp_session_t *openmrcp_session, openmrcp_event_t openmrcp_event) +static mrcp_status_t asr_session_signal_event(asr_session_t *asr_session, openmrcp_event_t openmrcp_event) { mrcp_status_t status = MRCP_STATUS_SUCCESS; switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Got event: %s\n", openmrcp_event_names[openmrcp_event]); // allocate memory for event - openmrcp_event_t *event2queue = (openmrcp_event_t *) switch_core_alloc(openmrcp_session->pool, sizeof(openmrcp_event_t)); + openmrcp_event_t *event2queue = (openmrcp_event_t *) switch_core_alloc(asr_session->pool, sizeof(openmrcp_event_t)); *event2queue = openmrcp_event; // add it to queue - if (switch_queue_trypush(openmrcp_session->event_queue, (void *) event2queue)) { + if (switch_queue_trypush(asr_session->event_queue, (void *) event2queue)) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "could not push event to queue\n"); status = SWITCH_STATUS_GENERR; } @@ -270,108 +329,262 @@ } -static mrcp_status_t openmrcp_on_session_initiate(mrcp_client_context_t *context, mrcp_session_t *session) +static mrcp_status_t asr_on_session_initiate(mrcp_client_context_t *context, mrcp_session_t *session) { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "openmrcp_on_session_initiate called\n"); - openmrcp_session_t *openmrcp_session = mrcp_client_context_session_object_get(session); - if(!openmrcp_session) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "asr_on_session_initiate called\n"); + asr_session_t *asr_session = mrcp_client_context_session_object_get(session); + if(!asr_session) { return MRCP_STATUS_FAILURE; } - return openmrcp_session_signal_event(openmrcp_session,OPENMRCP_EVENT_SESSION_INITIATE); + return asr_session_signal_event(asr_session,OPENMRCP_EVENT_SESSION_INITIATE); } -static mrcp_status_t openmrcp_on_session_terminate(mrcp_client_context_t *context, mrcp_session_t *session) +static mrcp_status_t asr_on_session_terminate(mrcp_client_context_t *context, mrcp_session_t *session) { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "openmrcp_on_session_terminate called\n"); - openmrcp_session_t *openmrcp_session = mrcp_client_context_session_object_get(session); - if(!openmrcp_session) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "asr_on_session_terminate called\n"); + asr_session_t *asr_session = mrcp_client_context_session_object_get(session); + if(!asr_session) { return MRCP_STATUS_FAILURE; } - return openmrcp_session_signal_event(openmrcp_session,OPENMRCP_EVENT_SESSION_TERMINATE); + return asr_session_signal_event(asr_session,OPENMRCP_EVENT_SESSION_TERMINATE); } -static mrcp_status_t openmrcp_on_channel_add(mrcp_client_context_t *context, mrcp_session_t *session, mrcp_client_channel_t *channel) +static mrcp_status_t asr_on_channel_add(mrcp_client_context_t *context, mrcp_session_t *session, mrcp_client_channel_t *channel) { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "openmrcp_on_channel_add called\n"); - openmrcp_session_t *openmrcp_session = mrcp_client_context_session_object_get(session); - if(!openmrcp_session) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "asr_on_channel_add called\n"); + asr_session_t *asr_session = mrcp_client_context_session_object_get(session); + if(!asr_session) { return MRCP_STATUS_FAILURE; } - return openmrcp_session_signal_event(openmrcp_session,OPENMRCP_EVENT_CHANNEL_CREATE); + return asr_session_signal_event(asr_session,OPENMRCP_EVENT_CHANNEL_CREATE); } -static mrcp_status_t openmrcp_on_channel_remove(mrcp_client_context_t *context, mrcp_session_t *session, mrcp_client_channel_t *channel) +static mrcp_status_t asr_on_channel_remove(mrcp_client_context_t *context, mrcp_session_t *session, mrcp_client_channel_t *channel) { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "openmrcp_on_channel_remove called\n"); - openmrcp_session_t *openmrcp_session = mrcp_client_context_session_object_get(session); - if(!openmrcp_session) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "asr_on_channel_remove called\n"); + asr_session_t *asr_session = mrcp_client_context_session_object_get(session); + if(!asr_session) { return MRCP_STATUS_FAILURE; } - return openmrcp_session_signal_event(openmrcp_session,OPENMRCP_EVENT_CHANNEL_DESTROY); + return asr_session_signal_event(asr_session,OPENMRCP_EVENT_CHANNEL_DESTROY); } /** this is called by the mrcp core whenever an mrcp message is received from the other side. */ -static mrcp_status_t openmrcp_on_channel_modify(mrcp_client_context_t *context, mrcp_session_t *session, mrcp_message_t *mrcp_message) +static mrcp_status_t asr_on_channel_modify(mrcp_client_context_t *context, mrcp_session_t *session, mrcp_message_t *mrcp_message) { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "openmrcp_on_channel_modify called\n"); - openmrcp_session_t *openmrcp_session = mrcp_client_context_session_object_get(session); - if(!openmrcp_session) { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "!openmrcp_session\n"); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "asr_on_channel_modify called\n"); + asr_session_t *asr_session = mrcp_client_context_session_object_get(session); + if(!asr_session) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "!asr_session\n"); return MRCP_STATUS_FAILURE; } switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "mrcp msg body: %s\n", mrcp_message->body); if (!strcmp(mrcp_message->start_line.method_name,"RECOGNITION-COMPLETE")) { - openmrcp_session->mrcp_message_last_rcvd = mrcp_message; + asr_session->mrcp_message_last_rcvd = mrcp_message; switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "setting FLAG_HAS_TEXT\n"); - switch_set_flag_locked(openmrcp_session, FLAG_HAS_TEXT); + switch_set_flag_locked(asr_session, FLAG_HAS_TEXT); } else if (!strcmp(mrcp_message->start_line.method_name,"START-OF-SPEECH")) { - openmrcp_session->mrcp_message_last_rcvd = mrcp_message; + asr_session->mrcp_message_last_rcvd = mrcp_message; switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "setting FLAG_BARGE\n"); - switch_set_flag_locked(openmrcp_session, FLAG_BARGE); + switch_set_flag_locked(asr_session, FLAG_BARGE); } else { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "ignoring method: %s\n", mrcp_message->start_line.method_name); } - return openmrcp_session_signal_event(openmrcp_session,OPENMRCP_EVENT_CHANNEL_MODIFY); + return asr_session_signal_event(asr_session,OPENMRCP_EVENT_CHANNEL_MODIFY); } -static apr_status_t set_default_options(openmrcp_client_options_t *options) +static mrcp_status_t tts_session_signal_event(tts_session_t *tts_session, openmrcp_event_t openmrcp_event) +{ + mrcp_status_t status = MRCP_STATUS_SUCCESS; + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Got event: %s\n", openmrcp_event_names[openmrcp_event]); + + // allocate memory for event + openmrcp_event_t *event2queue = (openmrcp_event_t *) switch_core_alloc(tts_session->pool, sizeof(openmrcp_event_t)); + *event2queue = openmrcp_event; + + // add it to queue + if (switch_queue_trypush(tts_session->event_queue, (void *) event2queue)) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "could not push event to queue\n"); + status = SWITCH_STATUS_GENERR; + } + else { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "pushed event to queue: %s\n", openmrcp_event_names[*event2queue]); + } + + return status; +} + +static mrcp_status_t tts_on_session_initiate(mrcp_client_context_t *context, mrcp_session_t *session) +{ + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "tts_on_session_initiate called\n"); + tts_session_t *tts_session = mrcp_client_context_session_object_get(session); + if(!tts_session) { + return MRCP_STATUS_FAILURE; + } + return tts_session_signal_event(tts_session,OPENMRCP_EVENT_SESSION_INITIATE); +} + +static mrcp_status_t tts_on_session_terminate(mrcp_client_context_t *context, mrcp_session_t *session) +{ + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "tts_on_session_terminate called\n"); + tts_session_t *tts_session = mrcp_client_context_session_object_get(session); + if(!tts_session) { + return MRCP_STATUS_FAILURE; + } + return tts_session_signal_event(tts_session,OPENMRCP_EVENT_SESSION_TERMINATE); +} + +static mrcp_status_t tts_on_channel_add(mrcp_client_context_t *context, mrcp_session_t *session, mrcp_client_channel_t *channel) +{ + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "tts_on_channel_add called\n"); + tts_session_t *tts_session = mrcp_client_context_session_object_get(session); + if(!tts_session) { + return MRCP_STATUS_FAILURE; + } + return tts_session_signal_event(tts_session,OPENMRCP_EVENT_CHANNEL_CREATE); +} + +static mrcp_status_t tts_on_channel_remove(mrcp_client_context_t *context, mrcp_session_t *session, mrcp_client_channel_t *channel) +{ + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "tts_on_channel_remove called\n"); + tts_session_t *tts_session = mrcp_client_context_session_object_get(session); + if(!tts_session) { + return MRCP_STATUS_FAILURE; + } + return tts_session_signal_event(tts_session,OPENMRCP_EVENT_CHANNEL_DESTROY); +} + +/** this is called by the mrcp core whenever an mrcp message is received from + the other side. */ +static mrcp_status_t tts_on_channel_modify(mrcp_client_context_t *context, mrcp_session_t *session, mrcp_message_t *mrcp_message) +{ + + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "tts_on_channel_modify called\n"); + tts_session_t *tts_session = mrcp_client_context_session_object_get(session); + if(!tts_session) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "!tts_session\n"); + return MRCP_STATUS_FAILURE; + } + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "mrcp msg body: %s\n", mrcp_message->body); + + return tts_session_signal_event(tts_session,OPENMRCP_EVENT_CHANNEL_MODIFY); +} + + + +static apr_status_t set_default_asr_options(openmrcp_client_options_t *options) +{ + mrcp_logger.priority = MRCP_PRIO_INFO; + options->proto_version = globals.asr_proto_version; + options->client_ip = globals.asr_client_ip; + options->server_ip = globals.asr_server_ip; + options->client_port = globals.asr_client_port; + options->server_port = globals.asr_server_port; + options->rtp_port_min = globals.rtp_port_min; + options->rtp_port_max = globals.rtp_port_max; + return APR_SUCCESS; +} + + +static apr_status_t set_default_tts_options(openmrcp_client_options_t *options) { mrcp_logger.priority = MRCP_PRIO_INFO; - options->proto_version = globals.proto_version; - options->client_ip = globals.client_ip; - options->server_ip = globals.server_ip; - options->client_port = globals.client_port; - options->server_port = globals.server_port; + options->proto_version = globals.tts_proto_version; + options->client_ip = globals.tts_client_ip; + options->server_ip = globals.tts_server_ip; + options->client_port = globals.tts_client_port; + options->server_port = globals.tts_server_port; options->rtp_port_min = globals.rtp_port_min; options->rtp_port_max = globals.rtp_port_max; return APR_SUCCESS; } +/** + * Called back by openmrcp client thread every time it receives audio + * from the TTS server we are connected to. Puts audio in a queueu + * and it will be pulled out from read_tts + */ +static apr_status_t openmrcp_tts_write_frame(audio_sink_t *sink, media_frame_t *frame) +{ + tts_session_t *tts_session = sink->object; + media_frame_t *media_frame; + switch_byte_t *buffer; + size_t len; + + len = frame->codec_frame.size; + + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "add_up frame: %li\n", + add_up_buffer(frame->codec_frame.buffer, len)); + + /* create new media frame */ + media_frame = (media_frame_t *) switch_core_alloc(tts_session->pool, sizeof(media_frame_t)); + if (!media_frame) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "media_frame creation failed\n"); + return SWITCH_STATUS_MEMERR; + } + + /** + * since *frame might get freed by caller (true or false?), allocate a + * new buffer and copy *data into it. + **/ + buffer = (switch_byte_t *) switch_core_alloc(tts_session->pool, sizeof(switch_byte_t)*len); + if (!buffer) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not allocate buffer\n"); + return SWITCH_STATUS_MEMERR; + } + buffer = memcpy(buffer, frame->codec_frame.buffer, len); + media_frame->codec_frame.buffer = buffer; + media_frame->codec_frame.size = len; + media_frame->type = MEDIA_FRAME_TYPE_AUDIO; + + + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "add_up media_frame: %li\n", + add_up_buffer(media_frame->codec_frame.buffer, len)); + + + /* push audio to queue */ + if (switch_queue_trypush(tts_session->audio_queue, (void *) media_frame)) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "could not push audio to queue\n"); + return MRCP_STATUS_FAILURE; + } + + return MRCP_STATUS_SUCCESS; + +} + +long add_up_buffer(switch_byte_t *buffer, size_t len) { + // add up contents in buffer + long result = 0; + for (int i=0; i<len; i++) { + result =+ buffer[i]; + } + return result; +} + /** - * Called back by openmcp client thread every time its ready for more audio - * Reads data that was put into a shared fifo queue upon receiving audio frames - * from asr_feed() + * Called back by openmcp client thread every time its ready for more audio to send + * the recognition server we are connected to. Reads data that was put into a + * shared fifo queue upon receiving audio frames from asr_feed() */ static apr_status_t openmrcp_recognizer_read_frame(audio_source_t *source, media_frame_t *frame) { - openmrcp_session_t *openmrcp_session = source->object; + asr_session_t *asr_session = source->object; frame->type = MEDIA_FRAME_TYPE_NONE; apr_status_t result; media_frame_t *queue_frame = NULL; - /* pop next media frame data from incoming queue into frame */ - if(openmrcp_session->audio_input_queue) { - if (switch_queue_trypop(openmrcp_session->audio_input_queue, (void *) &queue_frame)) { + if(asr_session->audio_queue) { + if (switch_queue_trypop(asr_session->audio_queue, (void *) &queue_frame)) { // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "could not pop from queue\n"); result = MRCP_STATUS_FAILURE; } @@ -383,7 +596,7 @@ } } else { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "no audio_in queue\n"); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "no audio queue\n"); result = MRCP_STATUS_FAILURE; } @@ -392,7 +605,7 @@ /** Read in the grammar and construct an MRCP Recognize message that has The grammar attached as the payload */ -static mrcp_status_t openmrcp_recog_start(mrcp_client_context_t *context, openmrcp_session_t *openmrcp_session, char *path) +static mrcp_status_t openmrcp_recog_start(mrcp_client_context_t *context, asr_session_t *asr_session, char *path) { mrcp_generic_header_t *generic_header; @@ -403,7 +616,7 @@ char *buf1; apr_size_t bytes2read = 0; - mrcp_message_t *mrcp_message = mrcp_client_context_message_get(context, openmrcp_session->client_session, openmrcp_session->channel, RECOGNIZER_RECOGNIZE); + mrcp_message_t *mrcp_message = mrcp_client_context_message_get(context, asr_session->client_session, asr_session->channel, RECOGNIZER_RECOGNIZE); if(!mrcp_message) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not create mrcp msg\n"); @@ -431,7 +644,7 @@ mrcp_message->body = mrcp_str_pdup(mrcp_message->pool,buf1); /* send the MRCP RECOGNIZE message to MRCP server */ - return mrcp_client_context_channel_modify(context, openmrcp_session->client_session, mrcp_message); + return mrcp_client_context_channel_modify(context, asr_session->client_session, mrcp_message); } @@ -446,8 +659,8 @@ */ static switch_status_t openmrcp_asr_open(switch_asr_handle_t *ah, char *codec, int rate, char *dest, switch_asr_flag_t *flags) { - mrcp_client_context_t *mrcp_client_context = openmrcp_module.mrcp_client_context ; - openmrcp_session_t *openmrcp_session; + mrcp_client_context_t *asr_client_context = openmrcp_module.asr_client_context ; + asr_session_t *asr_session; audio_source_t *source; switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "asr_open called, codec: %s, rate: %d\n", codec, rate); @@ -483,37 +696,37 @@ } /* create session */ - openmrcp_session = openmrcp_session_create(); - if (!openmrcp_session) { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "openmrcp_session creation FAILED\n"); + asr_session = asr_session_create(); + if (!asr_session) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "asr_session creation FAILED\n"); return SWITCH_STATUS_GENERR; } - openmrcp_session->client_session = mrcp_client_context_session_create(mrcp_client_context,openmrcp_session); - if (!openmrcp_session->client_session) { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "openmrcp_session creation FAILED\n"); + asr_session->client_session = mrcp_client_context_session_create(asr_client_context,asr_session); + if (!asr_session->client_session) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "asr_session creation FAILED\n"); return SWITCH_STATUS_GENERR; } /* create audio source */ - source = mrcp_palloc(openmrcp_session->pool,sizeof(audio_source_t)); + source = mrcp_palloc(asr_session->pool,sizeof(audio_source_t)); source->method_set = &audio_source_method_set; - source->object = openmrcp_session; - openmrcp_session->source = source; + source->object = asr_session; + asr_session->source = source; /** * create a new fifo queue. incoming audio received from freeswitch * will be put into this queue, and it will be later pulled out by * the openmrcp client thread. */ - if (switch_queue_create(&openmrcp_session->audio_input_queue, 10000, openmrcp_session->pool)) { + if (switch_queue_create(&asr_session->audio_queue, 10000, asr_session->pool)) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "audio queue creation failed\n"); return SWITCH_STATUS_MEMERR; } - openmrcp_session->flags = *flags; - switch_mutex_init(&openmrcp_session->flag_mutex, SWITCH_MUTEX_NESTED, openmrcp_session->pool); + asr_session->flags = *flags; + switch_mutex_init(&asr_session->flag_mutex, SWITCH_MUTEX_NESTED, asr_session->pool); - ah->private_info = openmrcp_session; + ah->private_info = asr_session; return SWITCH_STATUS_SUCCESS; @@ -528,28 +741,28 @@ TODO: - how does DEFINE-GRAMMAR fit into the picture here? (if at all) */ - openmrcp_session_t *openmrcp_session = (openmrcp_session_t *) ah->private_info; - mrcp_client_context_t *mrcp_client_context = openmrcp_module.mrcp_client_context; - audio_source_t *source = openmrcp_session->source; + asr_session_t *asr_session = (asr_session_t *) ah->private_info; + mrcp_client_context_t *asr_client_context = openmrcp_module.asr_client_context; + audio_source_t *source = asr_session->source; /* create recognizer channel, also starts outgoing rtp media */ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "Loading grammar\n"); switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Create Recognizer Channel\n"); - openmrcp_session->channel = mrcp_client_recognizer_channel_create(mrcp_client_context, openmrcp_session->client_session, source); + asr_session->channel = mrcp_client_recognizer_channel_create(asr_client_context, asr_session->client_session, source); - if (!openmrcp_session->channel) { + if (!asr_session->channel) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Failed to create recognizer channel\n"); return SWITCH_STATUS_FALSE; } /* wait for recognizer channel creation */ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "WAITING FOR CHAN CREATE\n"); - if(openmrcp_session_wait_for_event(openmrcp_session, OPENMRCP_EVENT_CHANNEL_CREATE, OPENMRCP_WAIT_TIMEOUT) == MRCP_STATUS_SUCCESS) { + if(wait_for_event(asr_session->event_queue, OPENMRCP_EVENT_CHANNEL_CREATE, OPENMRCP_WAIT_TIMEOUT) == MRCP_STATUS_SUCCESS) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Got channel creation event\n"); switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "WAITING FOR NONE\n"); - if (openmrcp_session_wait_for_event(openmrcp_session,OPENMRCP_EVENT_NONE,200) == MRCP_STATUS_SUCCESS) { + if (wait_for_event(asr_session->event_queue,OPENMRCP_EVENT_NONE,200) == MRCP_STATUS_SUCCESS) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "GOT NONE EVENT\n"); } else { @@ -557,7 +770,7 @@ } switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Start Recognizer\n"); - openmrcp_recog_start(mrcp_client_context, openmrcp_session, path); + openmrcp_recog_start(asr_client_context, asr_session, path); } else { @@ -573,12 +786,12 @@ static switch_status_t openmrcp_asr_feed(switch_asr_handle_t *ah, void *data, unsigned int len, switch_asr_flag_t *flags) { - openmrcp_session_t *openmrcp_session = (openmrcp_session_t *) ah->private_info; + asr_session_t *asr_session = (asr_session_t *) ah->private_info; media_frame_t *media_frame; switch_byte_t *buffer; /* create new media frame */ - media_frame = (media_frame_t *) switch_core_alloc(openmrcp_session->pool, sizeof(media_frame_t)); + media_frame = (media_frame_t *) switch_core_alloc(asr_session->pool, sizeof(media_frame_t)); if (!media_frame) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "media_frame creation failed\n"); return SWITCH_STATUS_MEMERR; @@ -592,7 +805,7 @@ * this needs to send the packet directly. OpenMrcp will need a way to disable their own * timer so that the fs timer is the only one driving the media. **/ - buffer = (switch_byte_t *) switch_core_alloc(openmrcp_session->pool, sizeof(switch_byte_t)*len); + buffer = (switch_byte_t *) switch_core_alloc(asr_session->pool, sizeof(switch_byte_t)*len); if (!buffer) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not allocate buffer\n"); return SWITCH_STATUS_MEMERR; @@ -604,7 +817,7 @@ media_frame->type = MEDIA_FRAME_TYPE_AUDIO; /* push audio to queue */ - if (switch_queue_trypush(openmrcp_session->audio_input_queue, (void *) media_frame)) { + if (switch_queue_trypush(asr_session->audio_queue, (void *) media_frame)) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "could not push audio to queue\n"); return SWITCH_STATUS_GENERR; } @@ -644,19 +857,19 @@ */ static switch_status_t openmrcp_asr_close(switch_asr_handle_t *ah, switch_asr_flag_t *flags) { - openmrcp_session_t *openmrcp_session = (openmrcp_session_t *) ah->private_info; + asr_session_t *asr_session = (asr_session_t *) ah->private_info; switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "openmrcp_asr_close()\n"); - mrcp_client_context_t *context = openmrcp_module.mrcp_client_context; + mrcp_client_context_t *context = openmrcp_module.asr_client_context; // TODO!! should we do a switch_pool_clear(switch_memory_pool_t *p) on the pool held - // by openmrcp_session? + // by asr_session? // destroy channel switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Going to DESTROY CHANNEL\n"); - mrcp_client_context_channel_destroy(context, openmrcp_session->client_session, openmrcp_session->channel); - if (openmrcp_session_wait_for_event(openmrcp_session,OPENMRCP_EVENT_CHANNEL_DESTROY,10000) == MRCP_STATUS_SUCCESS) { + mrcp_client_context_channel_destroy(context, asr_session->client_session, asr_session->channel); + if (wait_for_event(asr_session->event_queue,OPENMRCP_EVENT_CHANNEL_DESTROY,10000) == MRCP_STATUS_SUCCESS) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "OPENMRCP_EVENT_CHANNEL_DESTROY received\n"); } else { @@ -665,8 +878,8 @@ // terminate client session switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Going to TERMINATE SESSION\n"); - mrcp_client_context_session_terminate(context, openmrcp_session->client_session); - if (openmrcp_session_wait_for_event(openmrcp_session,OPENMRCP_EVENT_SESSION_TERMINATE,10000) == MRCP_STATUS_SUCCESS) { + mrcp_client_context_session_terminate(context, asr_session->client_session); + if (wait_for_event(asr_session->event_queue,OPENMRCP_EVENT_SESSION_TERMINATE,10000) == MRCP_STATUS_SUCCESS) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "OPENMRCP_EVENT_SESSION_TERMINATE recevied\n"); } else { @@ -675,10 +888,10 @@ // destroy client session (NOTE: this sends a BYE to the other side) switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Going to DESTROY SESSION\n"); - mrcp_client_context_session_destroy(context, openmrcp_session->client_session); + mrcp_client_context_session_destroy(context, asr_session->client_session); - // destroys the openmrcp_session struct - openmrcp_session_destroy(openmrcp_session); + // destroys the asr_session struct + asr_session_destroy(asr_session); switch_set_flag(ah, SWITCH_ASR_FLAG_CLOSED); @@ -694,9 +907,9 @@ */ static switch_status_t openmrcp_asr_check_results(switch_asr_handle_t *ah, switch_asr_flag_t *flags) { - openmrcp_session_t *openmrcp_session = (openmrcp_session_t *) ah->private_info; + asr_session_t *asr_session = (asr_session_t *) ah->private_info; - switch_status_t rv = (switch_test_flag(openmrcp_session, FLAG_HAS_TEXT) || switch_test_flag(openmrcp_session, FLAG_BARGE)) ? SWITCH_STATUS_SUCCESS : SWITCH_STATUS_FALSE; + switch_status_t rv = (switch_test_flag(asr_session, FLAG_HAS_TEXT) || switch_test_flag(asr_session, FLAG_BARGE)) ? SWITCH_STATUS_SUCCESS : SWITCH_STATUS_FALSE; return rv; } @@ -704,16 +917,16 @@ /*! This will be called after asr_check_results returns SUCCESS */ static switch_status_t openmrcp_asr_get_results(switch_asr_handle_t *ah, char **xmlstr, switch_asr_flag_t *flags) { - openmrcp_session_t *openmrcp_session = (openmrcp_session_t *) ah->private_info; + asr_session_t *asr_session = (asr_session_t *) ah->private_info; switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "openmrcp_asr_get_results called\n"); switch_status_t ret = SWITCH_STATUS_SUCCESS; - if (switch_test_flag(openmrcp_session, FLAG_BARGE)) { - switch_clear_flag_locked(openmrcp_session, FLAG_BARGE); + if (switch_test_flag(asr_session, FLAG_BARGE)) { + switch_clear_flag_locked(asr_session, FLAG_BARGE); ret = SWITCH_STATUS_BREAK; } - if (switch_test_flag(openmrcp_session, FLAG_HAS_TEXT)) { + if (switch_test_flag(asr_session, FLAG_HAS_TEXT)) { /*! we have to extract the XML but stripping off the <?xml version="1.0"?> header. the body looks like: @@ -724,16 +937,15 @@ <?xml version="1.0"?> <result xmlns="http://www.ietf.org/xml/ns/mrcpv2" xmlns:ex="http://www.example.com/example" score="100" grammar="session:request1@form-level.store"> - <interpretation> - <input mode="speech">open a</input> + <interpretation> <input mode="speech">open a</input> </interpretation> </result> */ char *marker = "?>"; // FIXME -- lame and brittle way of doing this. use regex or better. - char *position = strstr(openmrcp_session->mrcp_message_last_rcvd->body, marker); + char *position = strstr(asr_session->mrcp_message_last_rcvd->body, marker); if (!position) { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Bad result received from mrcp server: %s", openmrcp_session->mrcp_message_last_rcvd->body); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Bad result received from mrcp server: %s", asr_session->mrcp_message_last_rcvd->body); ret = SWITCH_STATUS_FALSE; } else { @@ -743,12 +955,236 @@ // since we are returning our result here, future calls to check_results // should return False - switch_clear_flag_locked(openmrcp_session, FLAG_HAS_TEXT); + switch_clear_flag_locked(asr_session, FLAG_HAS_TEXT); ret = SWITCH_STATUS_SUCCESS; } return ret; } + +static mrcp_status_t synth_speak(mrcp_client_context_t *context, tts_session_t *tts_session, char *text) +{ + + //buffer = (switch_byte_t *) switch_core_alloc(asr_session->pool, sizeof(switch_byte_t)*len); + char *text2speak; + const char xml_head[] = + "<?xml version=\"1.0\"?>\r\n" + "<speak>\r\n" + "<paragraph>\r\n" + " <sentence>"; + + const char xml_tail[] = "</sentence>\r\n" + "</paragraph>\r\n" + "</speak>\r\n"; + + size_t len = sizeof(xml_head) + sizeof(text) + sizeof(xml_tail); + text2speak = (char *) switch_core_alloc(tts_session->pool, len); + strcat(text2speak, xml_head); + strcat(text2speak, text); + strcat(text2speak, xml_tail); + + mrcp_generic_header_t *generic_header; + mrcp_message_t *mrcp_message = mrcp_client_context_message_get(context,tts_session->client_session,tts_session->channel,SYNTHESIZER_SPEAK); + if(!mrcp_message) { + return MRCP_STATUS_FAILURE; + } + + generic_header = mrcp_generic_header_prepare(mrcp_message); + if(!generic_header) { + return MRCP_STATUS_FAILURE; + } + + generic_header->content_type = mrcp_str_pdup(mrcp_message->pool,"application/synthesis+ssml"); + mrcp_generic_header_property_add(mrcp_message,GENERIC_HEADER_CONTENT_TYPE); + mrcp_message->body = mrcp_str_pdup(mrcp_message->pool,text2speak); + + return mrcp_client_context_channel_modify(context,tts_session->client_session,mrcp_message); +} + +static mrcp_status_t synth_stop(mrcp_client_context_t *context, tts_session_t *tts_session) +{ + mrcp_message_t *mrcp_message = mrcp_client_context_message_get(context,tts_session->client_session,tts_session->channel,SYNTHESIZER_STOP); + if(!mrcp_message) { + return MRCP_STATUS_FAILURE; + } + + return mrcp_client_context_channel_modify(context,tts_session->client_session,mrcp_message); +} + + +static switch_status_t openmrcp_tts_open(switch_speech_handle_t *sh, char *voice_name, int rate, switch_speech_flag_t *flags) +{ + + tts_session_t *tts_session; + audio_sink_t *sink; + mrcp_client_context_t *tts_client_context = openmrcp_module.tts_client_context ; + + /* create session */ + tts_session = tts_session_create(); + if (!tts_session) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "tts_session creation FAILED\n"); + return SWITCH_STATUS_GENERR; + } + tts_session->client_session = mrcp_client_context_session_create(tts_client_context,tts_session); + if (!tts_session->client_session) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "tts_session creation FAILED\n"); + return SWITCH_STATUS_GENERR; + } + + + /* create audio sink */ + sink = mrcp_palloc(tts_session->pool,sizeof(audio_sink_t)); + sink->method_set = &audio_sink_method_set; + sink->object = tts_session; + tts_session->sink = sink; + + + /** + * create a new fifo queue. + */ + if (switch_queue_create(&tts_session->audio_queue, 10000, tts_session->pool)) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "audio queue creation failed\n"); + return SWITCH_STATUS_MEMERR; + } + + + tts_session->flags = *flags; + switch_mutex_init(&tts_session->flag_mutex, SWITCH_MUTEX_NESTED, tts_session->pool); + + sh->private_info = tts_session; + + return SWITCH_STATUS_SUCCESS; + +} + +static switch_status_t openmrcp_tts_close(switch_speech_handle_t *sh, switch_speech_flag_t *flags) +{ + tts_session_t *tts_session = (tts_session_t *) sh->private_info; + mrcp_client_context_t *tts_client_context = openmrcp_module.tts_client_context ; + + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "synth_stop\n"); + synth_stop(tts_client_context,tts_session); // TODO + wait_for_event(tts_session->event_queue,OPENMRCP_EVENT_CHANNEL_MODIFY,5000); + + /* terminate tts session */ + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "Terminate tts_session\n"); + mrcp_client_context_session_terminate(tts_client_context,tts_session->client_session); + /* wait for tts session termination */ + wait_for_event(tts_session->event_queue,OPENMRCP_EVENT_SESSION_TERMINATE,10000); + + /* destroy demo session */ + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "destroy tts_session\n"); + mrcp_client_context_session_destroy(tts_client_context,tts_session->client_session); + tts_session_destroy(tts_session); + + return SWITCH_STATUS_SUCCESS; +} + +static switch_status_t openmrcp_feed_tts(switch_speech_handle_t *sh, char *text, switch_speech_flag_t *flags) +{ + + tts_session_t *tts_session = (tts_session_t *) sh->private_info; + mrcp_client_context_t *tts_client_context = openmrcp_module.tts_client_context ; + audio_sink_t *sink = tts_session->sink; + + /* create synthesizer channel */ + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "Create Synthesizer Channel\n"); + tts_session->channel = mrcp_client_synthesizer_channel_create(tts_client_context,tts_session->client_session,sink); + /* wait for synthesizer channel creation */ + if(wait_for_event(tts_session->event_queue,OPENMRCP_EVENT_CHANNEL_CREATE,5000) == MRCP_STATUS_SUCCESS) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "Got channel create event\n"); + wait_for_event(tts_session->event_queue,OPENMRCP_EVENT_NONE,1000); + /* speak */ + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "Going to speak\n"); + synth_speak(tts_client_context,tts_session, text); + // wait_for_event(tts_session->event_queue,OPENMRCP_EVENT_CHANNEL_MODIFY,5000); + + } + else { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Never got channel create event.\n"); + return SWITCH_STATUS_FALSE; + } + + + + return SWITCH_STATUS_SUCCESS; +} + +static void openmrcp_flush_tts(switch_speech_handle_t *sh) +{ + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "flush_tts called\n"); +} + +/** + * Freeswitch calls this when its ready to read datalen bytes of data. + * + * TODO: check the blocking flag and act accordingly + */ +static switch_status_t openmrcp_read_tts(switch_speech_handle_t *sh, void *data, size_t *datalen, uint32_t *rate, switch_speech_flag_t *flags) +{ + media_frame_t *queue_frame = NULL; + tts_session_t *tts_session = (tts_session_t *) sh->private_info; + size_t len = *datalen; + size_t x = 0; + + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "openmrcp_read_tts, datalen: %d\n", len); + + if (switch_queue_trypop(tts_session->audio_queue, (void *) &queue_frame)) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "could not pop from queue\n"); + //return SWITCH_STATUS_FALSE; + return SWITCH_STATUS_SUCCESS; + } + else { + + if (queue_frame->codec_frame.size != len) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "You must recompile openmrcp with #define CODEC_FRAME_TIME_BASE 20\n"); + return SWITCH_STATUS_FALSE; + } + + + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "add_up queue_frame: %li\n", + add_up_buffer(queue_frame->codec_frame.buffer, len)); + + + + switch_byte_t *bytedata = (switch_byte_t *) data; + switch_byte_t *sourcedata = (switch_byte_t *) queue_frame->codec_frame.buffer; + + // memcpy(data, queue_frame->codec_frame.buffer, len); + for (int i=0; i<len; i++) { + *(bytedata + i) = sourcedata[i]; + } + + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "add_up bytedata: %li\n", + add_up_buffer(bytedata, len)); + + + + // assume openmrcp is using 8kz for now + *rate = 8000; + + return SWITCH_STATUS_SUCCESS; + } + + + return SWITCH_STATUS_FALSE; +} + +static void openmrcp_text_param_tts(switch_speech_handle_t *sh, char *param, char *val) +{ + +} + +static void openmrcp_numeric_param_tts(switch_speech_handle_t *sh, char *param, int val) +{ + +} + +static void openmrcp_float_param_tts(switch_speech_handle_t *sh, char *param, double val) +{ + +} + static switch_asr_interface_t openmrcp_asr_interface = { /*.interface_name*/ "openmrcp", /*.asr_open*/ openmrcp_asr_open, @@ -762,6 +1198,19 @@ /*.asr_get_results*/ openmrcp_asr_get_results }; +static switch_speech_interface_t openmrcp_tts_interface = { + /*.interface_name*/ "openmrcp", + /*.speech_open*/ openmrcp_tts_open, + /*.speech_close*/ openmrcp_tts_close, + /*.speech_feed_tts*/ openmrcp_feed_tts, + /*.speech_read_tts*/ openmrcp_read_tts, + /*.speech_flush_tts*/ openmrcp_flush_tts, + /*.speech_text_param_tts*/ openmrcp_text_param_tts, + /*.speech_numeric_param_tts*/ openmrcp_numeric_param_tts, + /*.speech_float_param_tts*/ openmrcp_float_param_tts, + +}; + static switch_loadable_module_interface_t openmrcp_module_interface = { /*.module_name */ modname, /*.endpoint_interface */ NULL, @@ -771,7 +1220,7 @@ /*.application_interface */ NULL, /*.api_interface */ NULL, /*.file_interface */ NULL, - /*.speech_interface */ NULL, + /*.speech_interface */ &openmrcp_tts_interface, /*.directory_interface */ NULL, /*.chat_interface */ NULL, /*.say_interface */ NULL, @@ -796,16 +1245,26 @@ char *var = (char *) switch_xml_attr_soft(param, "name"); char *val = (char *) switch_xml_attr_soft(param, "value"); - if (!strcasecmp(var, "client_ip")) { - globals.client_ip = val; - } else if (!strcasecmp(var, "server_ip")) { - globals.server_ip = val; - } else if (!strcasecmp(var, "proto_version")) { - globals.proto_version =(uint32_t) atoi(val); - } else if (!strcasecmp(var, "client_port")) { - globals.client_port = (uint32_t) atoi(val); - } else if (!strcasecmp(var, "server_port")) { - globals.server_port = (uint32_t) atoi(val); + if (!strcasecmp(var, "asr_client_ip")) { + globals.asr_client_ip = val; + } else if (!strcasecmp(var, "asr_server_ip")) { + globals.asr_server_ip = val; + } else if (!strcasecmp(var, "asr_proto_version")) { + globals.asr_proto_version =(uint32_t) atoi(val); + } else if (!strcasecmp(var, "asr_client_port")) { + globals.asr_client_port = (uint32_t) atoi(val); + } else if (!strcasecmp(var, "asr_server_port")) { + globals.asr_server_port = (uint32_t) atoi(val); + } else if (!strcasecmp(var, "tts_client_ip")) { + globals.tts_client_ip = val; + } else if (!strcasecmp(var, "tts_server_ip")) { + globals.tts_server_ip = val; + } else if (!strcasecmp(var, "tts_proto_version")) { + globals.tts_proto_version =(uint32_t) atoi(val); + } else if (!strcasecmp(var, "tts_client_port")) { + globals.tts_client_port = (uint32_t) atoi(val); + } else if (!strcasecmp(var, "tts_server_port")) { + globals.tts_server_port = (uint32_t) atoi(val); } else if (!strcasecmp(var, "rtp_port_min")) { globals.rtp_port_min = (uint32_t) atoi(val); } else if (!strcasecmp(var, "rtp_port_max")) { @@ -822,44 +1281,86 @@ static switch_status_t mrcp_init() { /*! - Perform one-time initialization of client library + Perform one-time initialization of asr client library */ mrcp_mem_pool_t *pool; - mrcp_client_event_handler_t *client_event_handler; - mrcp_client_t *mrcp_client; - mrcp_client_context_t *mrcp_client_context; - openmrcp_client_options_t *options; + mrcp_client_event_handler_t *asr_event_handler; + mrcp_client_t *asr_client; + mrcp_client_context_t *asr_client_context; + openmrcp_client_options_t *asr_options; + + pool = mrcp_global_pool_get(); + asr_options = mrcp_palloc(pool,sizeof(openmrcp_client_options_t)); + asr_event_handler = mrcp_palloc(pool,sizeof(mrcp_client_event_handler_t)); + set_default_asr_options(asr_options); + + asr_event_handler->on_session_initiate = asr_on_session_initiate; + asr_event_handler->on_session_terminate = asr_on_session_terminate; + asr_event_handler->on_channel_add = asr_on_channel_add; + asr_event_handler->on_channel_remove = asr_on_channel_remove; + asr_event_handler->on_channel_modify = asr_on_channel_modify; + + // create asr client context, which to must be passed to client engine + asr_client_context = mrcp_client_context_create(&openmrcp_module,asr_event_handler); + if(!asr_client_context) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "asr_client_context creation failed\n"); + return SWITCH_STATUS_GENERR; + } + openmrcp_module.asr_client_context = asr_client_context; + + // this basically starts a thread that pulls events from the event queue + // and handles them + asr_client = openmrcp_client_start(asr_options,asr_client_context); + if(!asr_client) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "openasr_client_start FAILED\n"); + mrcp_client_context_destroy(asr_client_context); + return SWITCH_STATUS_GENERR; + } + + openmrcp_module.asr_client = asr_client; + + + /*! + Perform one-time initialization of tts client library + */ + + mrcp_client_event_handler_t *tts_event_handler; + mrcp_client_t *tts_client; + mrcp_client_context_t *tts_client_context; + openmrcp_client_options_t *tts_options; pool = mrcp_global_pool_get(); - options = mrcp_palloc(pool,sizeof(openmrcp_client_options_t)); - client_event_handler = mrcp_palloc(pool,sizeof(mrcp_client_event_handler_t)); - set_default_options(options); - - client_event_handler->on_session_initiate = openmrcp_on_session_initiate; - client_event_handler->on_session_terminate = openmrcp_on_session_terminate; - client_event_handler->on_channel_add = openmrcp_on_channel_add; - client_event_handler->on_channel_remove = openmrcp_on_channel_remove; - client_event_handler->on_channel_modify = openmrcp_on_channel_modify; - - // create client context, which to must be passed to client engine - mrcp_client_context = mrcp_client_context_create(&openmrcp_module,client_event_handler); - if(!mrcp_client_context) { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "mrcp_client_context creation failed\n"); + tts_options = mrcp_palloc(pool,sizeof(openmrcp_client_options_t)); + tts_event_handler = mrcp_palloc(pool,sizeof(mrcp_client_event_handler_t)); + set_default_tts_options(tts_options); + + tts_event_handler->on_session_initiate = tts_on_session_initiate; + tts_event_handler->on_session_terminate = tts_on_session_terminate; + tts_event_handler->on_channel_add = tts_on_channel_add; + tts_event_handler->on_channel_remove = tts_on_channel_remove; + tts_event_handler->on_channel_modify = tts_on_channel_modify; + + // create tts client context, which to must be passed to client engine + tts_client_context = mrcp_client_context_create(&openmrcp_module,tts_event_handler); + if(!tts_client_context) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "tts_client_context creation failed\n"); return SWITCH_STATUS_GENERR; } - openmrcp_module.mrcp_client_context = mrcp_client_context; + openmrcp_module.tts_client_context = tts_client_context; // this basically starts a thread that pulls events from the event queue // and handles them - mrcp_client = openmrcp_client_start(options,mrcp_client_context); - if(!mrcp_client) { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "openmrcp_client_start FAILED\n"); - mrcp_client_context_destroy(mrcp_client_context); + tts_client = openmrcp_client_start(tts_options,tts_client_context); + if(!tts_client) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "opentts_client_start FAILED\n"); + mrcp_client_context_destroy(tts_client_context); return SWITCH_STATUS_GENERR; } - openmrcp_module.mrcp_client = mrcp_client; + openmrcp_module.tts_client = tts_client; + + return SWITCH_STATUS_SUCCESS; }