August 2007 - freeswitch-branches@lists.freeswitch.org

[Freeswitch-branches] [commit] r5591 - freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp
by Freeswitch SVN 13 Aug '07

13 Aug '07

Author: greenlizard Date: Mon Aug 13 14:10:16 2007 New Revision: 5591 Modified: freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/mod_openmrcp.c Log: going to merge mod_openmrcp to trunk Modified: freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/mod_openmrcp.c ============================================================================== --- freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/mod_openmrcp.c (original) +++ freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/mod_openmrcp.c Mon Aug 13 14:10:16 2007 @@ -32,8 +32,29 @@ * * Uses OpenMrcp (http://wiki.freeswitch.org/wiki/OpenMRCP) as the * the client library. + * + * TODO + * ======= + * + * - MAJOR DESIGN ISSUE!! It is way too expensive to be calling malloc on every audio frame, + * this needs to send the packet directly. OpenMrcp will need a way to disable their own + * timer so that the fs timer is the only one driving the media. + * + * - There are two memory pools in use. One in asr_session which is managed + * by this module, and one in the switch_asr_handle_t, which is managed by freeswitch. + * These need to be consolidated into one. (basically throw away the one in asr_session) * + * - fs status codes (eg, SWITCH_STATUS_GENERR) and mrcp status codes (MRCP_STATUS_FAILURE) + * are intermixed badly. this needs cleanup + * + * - openmrcp_flush_tts, openmrcp_text_param_tts, openmrcp_numeric_param_tts, + * openmrcp_float_param_tts need to have functionality added + * + * - fix audio problem with TTS, convert from using queue to using a switch_buffer + * (in progress) + * */ + #ifdef __ICC #pragma warning (disable:188) #endif @@ -63,6 +84,8 @@ #include <switch.h> #define OPENMRCP_WAIT_TIMEOUT 5000 +#define MY_BUF_LEN 1024 * 32 +#define MY_BLOCK_SIZE MY_BUF_LEN SWITCH_MODULE_LOAD_FUNCTION(mod_openmrcp_load); SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_openmrcp_shutdown); @@ -120,16 +143,16 @@ struct tts_session_t { mrcp_session_t *client_session; mrcp_client_channel_t *channel; - switch_queue_t *audio_queue; + switch_queue_t *audio_queue; // TO BE REMOVED switch_queue_t *event_queue; - audio_sink_t *sink; + switch_mutex_t *audio_lock; + switch_buffer_t *audio_buffer; + audio_sink_t *sink; apr_pool_t *pool; switch_speech_flag_t flags; switch_mutex_t *flag_mutex; }; -long add_up_buffer(switch_byte_t *buffer, size_t len); // TEMP DEBUG - static apr_status_t openmrcp_recognizer_read_frame(audio_source_t *source, media_frame_t *frame); static apr_status_t openmrcp_tts_write_frame(audio_sink_t *sink, media_frame_t *frame); @@ -522,9 +545,6 @@ len = frame->codec_frame.size; - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "add_up frame: %li\n", - add_up_buffer(frame->codec_frame.buffer, len)); - /* create new media frame */ media_frame = (media_frame_t *) switch_core_alloc(tts_session->pool, sizeof(media_frame_t)); if (!media_frame) { @@ -547,10 +567,6 @@ media_frame->type = MEDIA_FRAME_TYPE_AUDIO; - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "add_up media_frame: %li\n", - add_up_buffer(media_frame->codec_frame.buffer, len)); - - /* push audio to queue */ if (switch_queue_trypush(tts_session->audio_queue, (void *) media_frame)) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "could not push audio to queue\n"); @@ -561,15 +577,63 @@ } -long add_up_buffer(switch_byte_t *buffer, size_t len) { - // add up contents in buffer - long result = 0; - for (int i=0; i<len; i++) { - result =+ buffer[i]; + +/** + * Called back by openmrcp client thread every time it receives audio + * from the TTS server we are connected to. Puts audio in a queueu + * and it will be pulled out from read_tts + */ +static apr_status_t openmrcp_tts_write_frame_NEW(audio_sink_t *sink, media_frame_t *frame) +{ + tts_session_t *tts_session = sink->object; + media_frame_t *media_frame; + switch_byte_t *buffer; + size_t len; + apr_status_t status = + + len = frame->codec_frame.size; + + /* create new media frame */ + media_frame = (media_frame_t *) switch_core_alloc(tts_session->pool, sizeof(media_frame_t)); + if (!media_frame) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "media_frame creation failed\n"); + return SWITCH_STATUS_MEMERR; } - return result; + + /** + * since *frame might get freed by caller (true or false?), allocate a + * new buffer and copy *data into it. + **/ + buffer = (switch_byte_t *) switch_core_alloc(tts_session->pool, sizeof(switch_byte_t)*len); + if (!buffer) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not allocate buffer\n"); + return SWITCH_STATUS_MEMERR; + } + buffer = memcpy(buffer, frame->codec_frame.buffer, len); + media_frame->codec_frame.buffer = buffer; + media_frame->codec_frame.size = len; + media_frame->type = MEDIA_FRAME_TYPE_AUDIO; + + + /* add audio to buffer */ + switch_mutex_lock(tts_session->audio_lock); + if (switch_buffer_write(tts_session->audio_buffer, buffer, len) == 0) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "Could not write to buffer\n"); + } + switch_mutex_unlock(tts_session->audio_lock); + + switch_mutex_unlock(tts_session->audio_lock); + + if (switch_queue_trypush(tts_session->audio_queue, (void *) media_frame)) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "could not push audio to queue\n"); + return MRCP_STATUS_FAILURE; + } + + return MRCP_STATUS_SUCCESS; + } + /** * Called back by openmcp client thread every time its ready for more audio to send * the recognition server we are connected to. Reads data that was put into a @@ -1047,6 +1111,15 @@ return SWITCH_STATUS_MEMERR; } + /* create mutex that will be used to lock audio buffer */ + switch_mutex_init(&tts_session->audio_lock, SWITCH_MUTEX_NESTED, sh->memory_pool); + + /* create audio buffer */ + if (switch_buffer_create_dynamic(&tts_session->audio_buffer, MY_BLOCK_SIZE, MY_BUF_LEN, 0) != SWITCH_STATUS_SUCCESS) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_CRIT, "Write Buffer Creation Failed!\n"); + return SWITCH_STATUS_MEMERR; + } + tts_session->flags = *flags; switch_mutex_init(&tts_session->flag_mutex, SWITCH_MUTEX_NESTED, tts_session->pool); @@ -1093,11 +1166,10 @@ /* wait for synthesizer channel creation */ if(wait_for_event(tts_session->event_queue,OPENMRCP_EVENT_CHANNEL_CREATE,5000) == MRCP_STATUS_SUCCESS) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "Got channel create event\n"); - wait_for_event(tts_session->event_queue,OPENMRCP_EVENT_NONE,1000); + wait_for_event(tts_session->event_queue,OPENMRCP_EVENT_NONE,1000); // XXX: what are we waiting for?? /* speak */ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "Going to speak\n"); synth_speak(tts_client_context,tts_session, text); - // wait_for_event(tts_session->event_queue,OPENMRCP_EVENT_CHANNEL_MODIFY,5000); } else { @@ -1105,20 +1177,14 @@ return SWITCH_STATUS_FALSE; } - - return SWITCH_STATUS_SUCCESS; } -static void openmrcp_flush_tts(switch_speech_handle_t *sh) -{ - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "flush_tts called\n"); -} - /** * Freeswitch calls this when its ready to read datalen bytes of data. * - * TODO: check the blocking flag and act accordingly + * TODO: check the blocking flag passed in flags and act accordingly + * (see mod_cepstral.c) */ static switch_status_t openmrcp_read_tts(switch_speech_handle_t *sh, void *data, size_t *datalen, uint32_t *rate, switch_speech_flag_t *flags) { @@ -1132,7 +1198,6 @@ return SWITCH_STATUS_SUCCESS; } else { - memcpy(data, queue_frame->codec_frame.buffer, queue_frame->codec_frame.size); *datalen = queue_frame->codec_frame.size; *rate = 8000; @@ -1141,6 +1206,12 @@ } + +static void openmrcp_flush_tts(switch_speech_handle_t *sh) +{ + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "flush_tts called\n"); +} + static void openmrcp_text_param_tts(switch_speech_handle_t *sh, char *param, char *val) {

1 0

[Freeswitch-branches] [commit] r5590 - freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp
by Freeswitch SVN 13 Aug '07

13 Aug '07

Author: greenlizard Date: Mon Aug 13 13:27:10 2007 New Revision: 5590 Modified: freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/mod_openmrcp.c Log: the tts stuff works now at the most basic level, however the audio sounds choppy since there is some discrepancy in the rtp packet sizes between openmrcp and what fs is expecting, and buffering in this module needs to be improved. Modified: freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/mod_openmrcp.c ============================================================================== --- freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/mod_openmrcp.c (original) +++ freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/mod_openmrcp.c Mon Aug 13 13:27:10 2007 @@ -1124,50 +1124,21 @@ { media_frame_t *queue_frame = NULL; tts_session_t *tts_session = (tts_session_t *) sh->private_info; - size_t len = *datalen; - size_t x = 0; - - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "openmrcp_read_tts, datalen: %d\n", len); + size_t requested_len = *datalen; + size_t return_len = 0; if (switch_queue_trypop(tts_session->audio_queue, (void *) &queue_frame)) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "could not pop from queue\n"); - //return SWITCH_STATUS_FALSE; return SWITCH_STATUS_SUCCESS; } else { - if (queue_frame->codec_frame.size != len) { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "You must recompile openmrcp with #define CODEC_FRAME_TIME_BASE 20\n"); - return SWITCH_STATUS_FALSE; - } - - - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "add_up queue_frame: %li\n", - add_up_buffer(queue_frame->codec_frame.buffer, len)); - - - - switch_byte_t *bytedata = (switch_byte_t *) data; - switch_byte_t *sourcedata = (switch_byte_t *) queue_frame->codec_frame.buffer; - - // memcpy(data, queue_frame->codec_frame.buffer, len); - for (int i=0; i<len; i++) { - *(bytedata + i) = sourcedata[i]; - } - - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "add_up bytedata: %li\n", - add_up_buffer(bytedata, len)); - - - - // assume openmrcp is using 8kz for now + memcpy(data, queue_frame->codec_frame.buffer, queue_frame->codec_frame.size); + *datalen = queue_frame->codec_frame.size; *rate = 8000; - return SWITCH_STATUS_SUCCESS; } - - return SWITCH_STATUS_FALSE; } static void openmrcp_text_param_tts(switch_speech_handle_t *sh, char *param, char *val)

1 0

[Freeswitch-branches] [commit] r5589 - freeswitch/branches/greenlizard/src/mod/languages/mod_python
by Freeswitch SVN 13 Aug '07

13 Aug '07

Author: greenlizard Date: Mon Aug 13 10:27:44 2007 New Revision: 5589 Modified: freeswitch/branches/greenlizard/src/mod/languages/mod_python/freeswitch_python.cpp Log: fix a bug introduced in last mod_python commit on greenlizard branch where dtmfs will segfault the switch due to an incorrect Py_XDECREF. fixed for dtmf handling, but event handling not retested yet Modified: freeswitch/branches/greenlizard/src/mod/languages/mod_python/freeswitch_python.cpp ============================================================================== --- freeswitch/branches/greenlizard/src/mod/languages/mod_python/freeswitch_python.cpp (original) +++ freeswitch/branches/greenlizard/src/mod/languages/mod_python/freeswitch_python.cpp Mon Aug 13 10:27:44 2007 @@ -109,7 +109,7 @@ PyObject *func, *arglist; PyObject *pyresult; - PyObject* headerdict; + PyObject *headerdict; char *resultStr; char *funcargs; @@ -159,6 +159,10 @@ headerdict = PyDict_New(); for (hp = event->headers; hp; hp = hp->next) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Adding event header to result"); + + // TODO: create PyObject pointers for name and value + // and explicitly decref them. all ref counting stuff is + // a mess and needs to be tested and looked at closer. PyDict_SetItem(headerdict, Py_BuildValue("s", hp->name), Py_BuildValue("s", hp->value)); @@ -172,7 +176,9 @@ "headers", headerdict); - //arglist = Py_BuildValue("(sis)", event->body, itype, funcargs); + Py_XDECREF(headerdict); + + } else { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Unknown input type: %d\n", itype); @@ -189,10 +195,9 @@ did_swap_in = end_allow_threads(); pyresult = PyEval_CallObject(func, arglist); - + Py_XDECREF(arglist); // Trash arglist - Py_XDECREF(headerdict); if (pyresult && pyresult != Py_None) { resultStr = (char *) PyString_AsString(pyresult); @@ -200,7 +205,7 @@ return cbresult; } else { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Python callback\n returned None"); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "Python callback\n returned None"); PyErr_Print(); PyErr_Clear(); }

1 0

[Freeswitch-branches] [commit] r5588 - freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp
by Freeswitch SVN 11 Aug '07

11 Aug '07

Author: greenlizard Date: Sat Aug 11 19:38:38 2007 New Revision: 5588 Modified: freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/Makefile freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/mod_openmrcp.c Log: started the tts functionality integration, still not working because there is no audio. Modified: freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/Makefile ============================================================================== --- freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/Makefile (original) +++ freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/Makefile Sat Aug 11 19:38:38 2007 @@ -8,18 +8,18 @@ # -Werror from the ALL_CFLAGS variable. # TODO -# - fix all the code that is causing warnings (We are currently working with Joe Stenbrenner from Division Management to have this task outsourced to Bangalore) -# - re-enable the -Werror flag (see above) +# - re-enable the -Werror flag after openmrcp compilation warnings are fixed # - needs to compile/link against the _freeswitch_ sofia libs (see FIXME) -# - is there an openmrcp dir in the fs tree yet? (see FIXME) +# - Should this download and build openmrcp? What's the plan? (see FIXME) -# FIXME!!! Should this download and build openmrcp? What's the plan? +# FIXME!!! OPENMRCP_DIR=/usr/src/openmrcp_trunk OPENMRCP_INCLUDE=-I$(OPENMRCP_DIR)/mrcpcore/client/include/ -I$(OPENMRCP_DIR)/platform/openmrcpclient/include/ -I$(OPENMRCP_DIR)/mediaframe/include/ -I$(OPENMRCP_DIR)/mrcpcore/engine/include/ -I$(OPENMRCP_DIR)/mrcpcore/include/ -I$(OPENMRCP_DIR)/mrcpcore/parser/include/ -I$(OPENMRCP_DIR)/mrcpcore/server/include/ -I$(OPENMRCP_DIR)/mrcpcore/media/include/ -I$(OPENMRCP_DIR)/mrcpcore/util/include -I$(OPENMRCP_DIR)/mrcpcore/resource/include/ OPENMRCP_LIBS_TRUNK=$(OPENMRCP_DIR)/platform/openmrcpclient/.libs/libopenmrcpclient.a $(OPENMRCP_DIR)/platform/openmrcpserver/.libs/libopenmrcpserver.a $(OPENMRCP_DIR)/mrcpcore/client/.libs/libmrcpclient.a $(OPENMRCP_DIR)/mediaframe/.libs/libmediaframe.a $(OPENMRCP_DIR)/mrcpcore/engine/.libs/libmrcpengine.a $(OPENMRCP_DIR)/mrcpcore/parser/.libs/libmrcpparser.a $(OPENMRCP_DIR)/mrcpcore/server/.libs/libmrcpserver.a $(OPENMRCP_DIR)/mrcpcore/media/.libs/libmrcpmedia.a $(OPENMRCP_DIR)/mrcpcore/util/.libs/libmrcputil.a $(OPENMRCP_DIR)/mrcpcore/util/.libs/libmrcputil.a $(OPENMRCP_DIR)/mrcpcore/resource/.libs/libmrcpresource.a $(OPENMRCP_DIR)/module/plugin/demo/.libs/libdemoplugin.a $(OPENMRCP_DIR)/module/mrcpv2agent/.libs/libmrcpv2agent.a $(OPENMRCP_DIR)/module/mrcpmediaframe/.libs/libmrcpmediaframe.a $(OPENMRCP_DIR)/module/mrcpsofiasip/.libs/libmrcpsofiasip.a $(OPENMRCP_DIR)/mrcpcore/.libs/libmrcpcore.a +# can remove this .. OPENMRCP_LIBS_STABLE=$(OPENMRCP_DIR)/platform/openmrcpclient/.libs/libopenmrcpclient.a $(OPENMRCP_DIR)/platform/openmrcpserver/.libs/libopenmrcpserver.a $(OPENMRCP_DIR)/mrcpcore/client/.libs/libmrcpclient.a $(OPENMRCP_DIR)/mediaframe/.libs/libmediaframe.a $(OPENMRCP_DIR)/mrcpcore/engine/.libs/libmrcpengine.a $(OPENMRCP_DIR)/mrcpcore/parser/.libs/libmrcpparser.a $(OPENMRCP_DIR)/mrcpcore/server/.libs/libmrcpserver.a $(OPENMRCP_DIR)/mrcpcore/media/.libs/libmrcpmedia.a $(OPENMRCP_DIR)/mrcpcore/util/.libs/libmrcputil.a $(OPENMRCP_DIR)/mrcpcore/util/.libs/libmrcputil.a $(OPENMRCP_DIR)/mrcpcore/resource/.libs/libmrcpresource.a $(OPENMRCP_DIR)/module/plugin/demo/.libs/libdemoplugin.a $(OPENMRCP_DIR)/module/mrcpv2agent/.libs/libmrcpv2agent.a $(OPENMRCP_DIR)/module/mrcplivemedia/.libs/libmrcplivemedia.a $(OPENMRCP_DIR)/module/mrcpsofiasip/.libs/libmrcpsofiasip.a $(OPENMRCP_DIR)/mrcpcore/.libs/libmrcpcore.a OPENMRCP_LIBS=$(OPENMRCP_LIBS_TRUNK) Modified: freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/mod_openmrcp.c ============================================================================== --- freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/mod_openmrcp.c (original) +++ freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/mod_openmrcp.c Sat Aug 11 19:38:38 2007 @@ -42,6 +42,7 @@ #include "openmrcp_client.h" #include "mrcp_client_context.h" #include "mrcp_recognizer.h" +#include "mrcp_synthesizer.h" #include "mrcp_generic_header.h" #include "rtp_session.h" #include "mrcp_client.h" @@ -69,11 +70,16 @@ mod_openmrcp_shutdown, NULL); static struct { - char *client_ip; - char *server_ip; - uint32_t proto_version; - uint32_t client_port; - uint32_t server_port; + char *asr_client_ip; + char *asr_server_ip; + uint32_t asr_proto_version; + uint32_t asr_client_port; + uint32_t asr_server_port; + char *tts_client_ip; + char *tts_server_ip; + uint32_t tts_proto_version; + uint32_t tts_client_port; + uint32_t tts_server_port; uint32_t rtp_port_min; uint32_t rtp_port_max; } globals; @@ -97,24 +103,35 @@ "CHANNEL_MODIFY", }; -// TODO: rename this to mod_openmrcp_session_t -typedef struct openmrcp_session_t openmrcp_session_t; -struct openmrcp_session_t { +typedef struct asr_session_t asr_session_t; +struct asr_session_t { mrcp_session_t *client_session; mrcp_client_channel_t *channel; - switch_queue_t *audio_input_queue; + switch_queue_t *audio_queue; switch_queue_t *event_queue; mrcp_message_t *mrcp_message_last_rcvd; audio_source_t *source; apr_pool_t *pool; - apr_thread_cond_t *wait_object; - apr_thread_mutex_t *wait_object_mutex; - openmrcp_event_t wait_event; uint32_t flags; switch_mutex_t *flag_mutex; }; +typedef struct tts_session_t tts_session_t; +struct tts_session_t { + mrcp_session_t *client_session; + mrcp_client_channel_t *channel; + switch_queue_t *audio_queue; + switch_queue_t *event_queue; + audio_sink_t *sink; + apr_pool_t *pool; + switch_speech_flag_t flags; + switch_mutex_t *flag_mutex; +}; + +long add_up_buffer(switch_byte_t *buffer, size_t len); // TEMP DEBUG + static apr_status_t openmrcp_recognizer_read_frame(audio_source_t *source, media_frame_t *frame); +static apr_status_t openmrcp_tts_write_frame(audio_sink_t *sink, media_frame_t *frame); static const audio_source_method_set_t audio_source_method_set = { NULL, @@ -123,6 +140,13 @@ openmrcp_recognizer_read_frame }; +static const audio_sink_method_set_t audio_sink_method_set = { + NULL, + NULL, + NULL, + openmrcp_tts_write_frame +}; + typedef enum { FLAG_HAS_TEXT = (1 << 0), FLAG_BARGE = (1 << 1), @@ -130,15 +154,17 @@ } mrcp_flag_t; typedef struct { - mrcp_client_t *mrcp_client; - mrcp_client_context_t *mrcp_client_context; + mrcp_client_t *asr_client; + mrcp_client_context_t *asr_client_context; + mrcp_client_t *tts_client; + mrcp_client_context_t *tts_client_context; } openmrcp_module_t; static openmrcp_module_t openmrcp_module; -static openmrcp_session_t* openmrcp_session_create() +static asr_session_t* asr_session_create() { - openmrcp_session_t *openmrcp_session; + asr_session_t *asr_session; apr_pool_t *session_pool; if(apr_pool_create(&session_pool,NULL) != APR_SUCCESS) { @@ -146,52 +172,85 @@ return NULL; } - openmrcp_session = apr_palloc(session_pool,sizeof(openmrcp_session_t)); - openmrcp_session->pool = session_pool; - openmrcp_session->client_session = NULL; - openmrcp_session->channel = NULL; - openmrcp_session->audio_input_queue = NULL; - openmrcp_session->event_queue = NULL; - openmrcp_session->wait_event = OPENMRCP_EVENT_NONE; + asr_session = apr_palloc(session_pool,sizeof(asr_session_t)); + asr_session->pool = session_pool; + asr_session->client_session = NULL; + asr_session->channel = NULL; + asr_session->audio_queue = NULL; + asr_session->event_queue = NULL; - apr_thread_mutex_create(&openmrcp_session->wait_object_mutex, APR_THREAD_MUTEX_UNNESTED, openmrcp_session->pool); - apr_thread_cond_create(&openmrcp_session->wait_object, openmrcp_session->pool); + /* create an event queue */ + if (switch_queue_create(&asr_session->event_queue, 1000, asr_session->pool)) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "event queue creation failed\n"); + } + return asr_session; +} + + +static tts_session_t* tts_session_create() +{ + tts_session_t *tts_session; + apr_pool_t *session_pool; + + if(apr_pool_create(&session_pool,NULL) != APR_SUCCESS) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "failed to create session_pool\n"); + return NULL; + } + + tts_session = apr_palloc(session_pool,sizeof(tts_session_t)); + tts_session->pool = session_pool; + tts_session->client_session = NULL; + tts_session->channel = NULL; + tts_session->audio_queue = NULL; + tts_session->event_queue = NULL; /* create an event queue */ - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "creating event queue\n"); - if (switch_queue_create(&openmrcp_session->event_queue, 1000, openmrcp_session->pool)) { + if (switch_queue_create(&tts_session->event_queue, 1000, tts_session->pool)) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "event queue creation failed\n"); } - return openmrcp_session; + return tts_session; +} + + +static mrcp_status_t asr_session_destroy(asr_session_t *asr_session) +{ + if(!asr_session) { + return MRCP_STATUS_FAILURE; + } + + if(asr_session->pool) { + apr_pool_destroy(asr_session->pool); + asr_session->pool = NULL; + } + return MRCP_STATUS_SUCCESS; } -static mrcp_status_t openmrcp_session_destroy(openmrcp_session_t *openmrcp_session) + +static mrcp_status_t tts_session_destroy(tts_session_t *tts_session) { - if(!openmrcp_session) { + if(!tts_session) { return MRCP_STATUS_FAILURE; } - if(openmrcp_session->pool) { - apr_thread_cond_destroy(openmrcp_session->wait_object); - apr_thread_mutex_destroy(openmrcp_session->wait_object_mutex); - apr_pool_destroy(openmrcp_session->pool); - openmrcp_session->pool = NULL; + if(tts_session->pool) { + apr_pool_destroy(tts_session->pool); + tts_session->pool = NULL; } return MRCP_STATUS_SUCCESS; } -static mrcp_status_t openmrcp_session_wait_for_event(openmrcp_session_t *openmrcp_session, openmrcp_event_t openmrcp_event, size_t timeout) +static mrcp_status_t wait_for_event(switch_queue_t *event_queue, openmrcp_event_t openmrcp_event, size_t timeout) { openmrcp_event_t *popped_event = NULL; size_t sleep_ms = 100; - if(openmrcp_session->event_queue) { - if (switch_queue_trypop(openmrcp_session->event_queue, (void *) &popped_event)) { + if(event_queue) { + if (switch_queue_trypop(event_queue, (void *) &popped_event)) { // most likely this failed because queue is empty. sleep for timeout seconds and try again? if (timeout > 0) { if (timeout < sleep_ms) { @@ -206,7 +265,7 @@ // call recursively // TODO: This is going to end up in a lot of recursion and // maybe blow the stack .. rethink this approach - return openmrcp_session_wait_for_event(openmrcp_session, openmrcp_event, timeout); + return wait_for_event(event_queue, openmrcp_event, timeout); } else { @@ -234,7 +293,7 @@ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "popped name: %s\n", openmrcp_event_names[*popped_event]); switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "popped [%s] but was expecting [%s], but maybe there's other things in queue, or something will arrive\n", openmrcp_event_names[*popped_event], openmrcp_event_names[openmrcp_event]); switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "calling recursively\n"); - return openmrcp_session_wait_for_event(openmrcp_session, openmrcp_event, timeout); + return wait_for_event(event_queue, openmrcp_event, timeout); } } @@ -247,18 +306,18 @@ } -static mrcp_status_t openmrcp_session_signal_event(openmrcp_session_t *openmrcp_session, openmrcp_event_t openmrcp_event) +static mrcp_status_t asr_session_signal_event(asr_session_t *asr_session, openmrcp_event_t openmrcp_event) { mrcp_status_t status = MRCP_STATUS_SUCCESS; switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Got event: %s\n", openmrcp_event_names[openmrcp_event]); // allocate memory for event - openmrcp_event_t *event2queue = (openmrcp_event_t *) switch_core_alloc(openmrcp_session->pool, sizeof(openmrcp_event_t)); + openmrcp_event_t *event2queue = (openmrcp_event_t *) switch_core_alloc(asr_session->pool, sizeof(openmrcp_event_t)); *event2queue = openmrcp_event; // add it to queue - if (switch_queue_trypush(openmrcp_session->event_queue, (void *) event2queue)) { + if (switch_queue_trypush(asr_session->event_queue, (void *) event2queue)) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "could not push event to queue\n"); status = SWITCH_STATUS_GENERR; } @@ -270,108 +329,262 @@ } -static mrcp_status_t openmrcp_on_session_initiate(mrcp_client_context_t *context, mrcp_session_t *session) +static mrcp_status_t asr_on_session_initiate(mrcp_client_context_t *context, mrcp_session_t *session) { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "openmrcp_on_session_initiate called\n"); - openmrcp_session_t *openmrcp_session = mrcp_client_context_session_object_get(session); - if(!openmrcp_session) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "asr_on_session_initiate called\n"); + asr_session_t *asr_session = mrcp_client_context_session_object_get(session); + if(!asr_session) { return MRCP_STATUS_FAILURE; } - return openmrcp_session_signal_event(openmrcp_session,OPENMRCP_EVENT_SESSION_INITIATE); + return asr_session_signal_event(asr_session,OPENMRCP_EVENT_SESSION_INITIATE); } -static mrcp_status_t openmrcp_on_session_terminate(mrcp_client_context_t *context, mrcp_session_t *session) +static mrcp_status_t asr_on_session_terminate(mrcp_client_context_t *context, mrcp_session_t *session) { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "openmrcp_on_session_terminate called\n"); - openmrcp_session_t *openmrcp_session = mrcp_client_context_session_object_get(session); - if(!openmrcp_session) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "asr_on_session_terminate called\n"); + asr_session_t *asr_session = mrcp_client_context_session_object_get(session); + if(!asr_session) { return MRCP_STATUS_FAILURE; } - return openmrcp_session_signal_event(openmrcp_session,OPENMRCP_EVENT_SESSION_TERMINATE); + return asr_session_signal_event(asr_session,OPENMRCP_EVENT_SESSION_TERMINATE); } -static mrcp_status_t openmrcp_on_channel_add(mrcp_client_context_t *context, mrcp_session_t *session, mrcp_client_channel_t *channel) +static mrcp_status_t asr_on_channel_add(mrcp_client_context_t *context, mrcp_session_t *session, mrcp_client_channel_t *channel) { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "openmrcp_on_channel_add called\n"); - openmrcp_session_t *openmrcp_session = mrcp_client_context_session_object_get(session); - if(!openmrcp_session) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "asr_on_channel_add called\n"); + asr_session_t *asr_session = mrcp_client_context_session_object_get(session); + if(!asr_session) { return MRCP_STATUS_FAILURE; } - return openmrcp_session_signal_event(openmrcp_session,OPENMRCP_EVENT_CHANNEL_CREATE); + return asr_session_signal_event(asr_session,OPENMRCP_EVENT_CHANNEL_CREATE); } -static mrcp_status_t openmrcp_on_channel_remove(mrcp_client_context_t *context, mrcp_session_t *session, mrcp_client_channel_t *channel) +static mrcp_status_t asr_on_channel_remove(mrcp_client_context_t *context, mrcp_session_t *session, mrcp_client_channel_t *channel) { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "openmrcp_on_channel_remove called\n"); - openmrcp_session_t *openmrcp_session = mrcp_client_context_session_object_get(session); - if(!openmrcp_session) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "asr_on_channel_remove called\n"); + asr_session_t *asr_session = mrcp_client_context_session_object_get(session); + if(!asr_session) { return MRCP_STATUS_FAILURE; } - return openmrcp_session_signal_event(openmrcp_session,OPENMRCP_EVENT_CHANNEL_DESTROY); + return asr_session_signal_event(asr_session,OPENMRCP_EVENT_CHANNEL_DESTROY); } /** this is called by the mrcp core whenever an mrcp message is received from the other side. */ -static mrcp_status_t openmrcp_on_channel_modify(mrcp_client_context_t *context, mrcp_session_t *session, mrcp_message_t *mrcp_message) +static mrcp_status_t asr_on_channel_modify(mrcp_client_context_t *context, mrcp_session_t *session, mrcp_message_t *mrcp_message) { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "openmrcp_on_channel_modify called\n"); - openmrcp_session_t *openmrcp_session = mrcp_client_context_session_object_get(session); - if(!openmrcp_session) { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "!openmrcp_session\n"); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "asr_on_channel_modify called\n"); + asr_session_t *asr_session = mrcp_client_context_session_object_get(session); + if(!asr_session) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "!asr_session\n"); return MRCP_STATUS_FAILURE; } switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "mrcp msg body: %s\n", mrcp_message->body); if (!strcmp(mrcp_message->start_line.method_name,"RECOGNITION-COMPLETE")) { - openmrcp_session->mrcp_message_last_rcvd = mrcp_message; + asr_session->mrcp_message_last_rcvd = mrcp_message; switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "setting FLAG_HAS_TEXT\n"); - switch_set_flag_locked(openmrcp_session, FLAG_HAS_TEXT); + switch_set_flag_locked(asr_session, FLAG_HAS_TEXT); } else if (!strcmp(mrcp_message->start_line.method_name,"START-OF-SPEECH")) { - openmrcp_session->mrcp_message_last_rcvd = mrcp_message; + asr_session->mrcp_message_last_rcvd = mrcp_message; switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "setting FLAG_BARGE\n"); - switch_set_flag_locked(openmrcp_session, FLAG_BARGE); + switch_set_flag_locked(asr_session, FLAG_BARGE); } else { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "ignoring method: %s\n", mrcp_message->start_line.method_name); } - return openmrcp_session_signal_event(openmrcp_session,OPENMRCP_EVENT_CHANNEL_MODIFY); + return asr_session_signal_event(asr_session,OPENMRCP_EVENT_CHANNEL_MODIFY); } -static apr_status_t set_default_options(openmrcp_client_options_t *options) +static mrcp_status_t tts_session_signal_event(tts_session_t *tts_session, openmrcp_event_t openmrcp_event) +{ + mrcp_status_t status = MRCP_STATUS_SUCCESS; + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Got event: %s\n", openmrcp_event_names[openmrcp_event]); + + // allocate memory for event + openmrcp_event_t *event2queue = (openmrcp_event_t *) switch_core_alloc(tts_session->pool, sizeof(openmrcp_event_t)); + *event2queue = openmrcp_event; + + // add it to queue + if (switch_queue_trypush(tts_session->event_queue, (void *) event2queue)) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "could not push event to queue\n"); + status = SWITCH_STATUS_GENERR; + } + else { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "pushed event to queue: %s\n", openmrcp_event_names[*event2queue]); + } + + return status; +} + +static mrcp_status_t tts_on_session_initiate(mrcp_client_context_t *context, mrcp_session_t *session) +{ + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "tts_on_session_initiate called\n"); + tts_session_t *tts_session = mrcp_client_context_session_object_get(session); + if(!tts_session) { + return MRCP_STATUS_FAILURE; + } + return tts_session_signal_event(tts_session,OPENMRCP_EVENT_SESSION_INITIATE); +} + +static mrcp_status_t tts_on_session_terminate(mrcp_client_context_t *context, mrcp_session_t *session) +{ + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "tts_on_session_terminate called\n"); + tts_session_t *tts_session = mrcp_client_context_session_object_get(session); + if(!tts_session) { + return MRCP_STATUS_FAILURE; + } + return tts_session_signal_event(tts_session,OPENMRCP_EVENT_SESSION_TERMINATE); +} + +static mrcp_status_t tts_on_channel_add(mrcp_client_context_t *context, mrcp_session_t *session, mrcp_client_channel_t *channel) +{ + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "tts_on_channel_add called\n"); + tts_session_t *tts_session = mrcp_client_context_session_object_get(session); + if(!tts_session) { + return MRCP_STATUS_FAILURE; + } + return tts_session_signal_event(tts_session,OPENMRCP_EVENT_CHANNEL_CREATE); +} + +static mrcp_status_t tts_on_channel_remove(mrcp_client_context_t *context, mrcp_session_t *session, mrcp_client_channel_t *channel) +{ + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "tts_on_channel_remove called\n"); + tts_session_t *tts_session = mrcp_client_context_session_object_get(session); + if(!tts_session) { + return MRCP_STATUS_FAILURE; + } + return tts_session_signal_event(tts_session,OPENMRCP_EVENT_CHANNEL_DESTROY); +} + +/** this is called by the mrcp core whenever an mrcp message is received from + the other side. */ +static mrcp_status_t tts_on_channel_modify(mrcp_client_context_t *context, mrcp_session_t *session, mrcp_message_t *mrcp_message) +{ + + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "tts_on_channel_modify called\n"); + tts_session_t *tts_session = mrcp_client_context_session_object_get(session); + if(!tts_session) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "!tts_session\n"); + return MRCP_STATUS_FAILURE; + } + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "mrcp msg body: %s\n", mrcp_message->body); + + return tts_session_signal_event(tts_session,OPENMRCP_EVENT_CHANNEL_MODIFY); +} + + + +static apr_status_t set_default_asr_options(openmrcp_client_options_t *options) +{ + mrcp_logger.priority = MRCP_PRIO_INFO; + options->proto_version = globals.asr_proto_version; + options->client_ip = globals.asr_client_ip; + options->server_ip = globals.asr_server_ip; + options->client_port = globals.asr_client_port; + options->server_port = globals.asr_server_port; + options->rtp_port_min = globals.rtp_port_min; + options->rtp_port_max = globals.rtp_port_max; + return APR_SUCCESS; +} + + +static apr_status_t set_default_tts_options(openmrcp_client_options_t *options) { mrcp_logger.priority = MRCP_PRIO_INFO; - options->proto_version = globals.proto_version; - options->client_ip = globals.client_ip; - options->server_ip = globals.server_ip; - options->client_port = globals.client_port; - options->server_port = globals.server_port; + options->proto_version = globals.tts_proto_version; + options->client_ip = globals.tts_client_ip; + options->server_ip = globals.tts_server_ip; + options->client_port = globals.tts_client_port; + options->server_port = globals.tts_server_port; options->rtp_port_min = globals.rtp_port_min; options->rtp_port_max = globals.rtp_port_max; return APR_SUCCESS; } +/** + * Called back by openmrcp client thread every time it receives audio + * from the TTS server we are connected to. Puts audio in a queueu + * and it will be pulled out from read_tts + */ +static apr_status_t openmrcp_tts_write_frame(audio_sink_t *sink, media_frame_t *frame) +{ + tts_session_t *tts_session = sink->object; + media_frame_t *media_frame; + switch_byte_t *buffer; + size_t len; + + len = frame->codec_frame.size; + + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "add_up frame: %li\n", + add_up_buffer(frame->codec_frame.buffer, len)); + + /* create new media frame */ + media_frame = (media_frame_t *) switch_core_alloc(tts_session->pool, sizeof(media_frame_t)); + if (!media_frame) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "media_frame creation failed\n"); + return SWITCH_STATUS_MEMERR; + } + + /** + * since *frame might get freed by caller (true or false?), allocate a + * new buffer and copy *data into it. + **/ + buffer = (switch_byte_t *) switch_core_alloc(tts_session->pool, sizeof(switch_byte_t)*len); + if (!buffer) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not allocate buffer\n"); + return SWITCH_STATUS_MEMERR; + } + buffer = memcpy(buffer, frame->codec_frame.buffer, len); + media_frame->codec_frame.buffer = buffer; + media_frame->codec_frame.size = len; + media_frame->type = MEDIA_FRAME_TYPE_AUDIO; + + + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "add_up media_frame: %li\n", + add_up_buffer(media_frame->codec_frame.buffer, len)); + + + /* push audio to queue */ + if (switch_queue_trypush(tts_session->audio_queue, (void *) media_frame)) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "could not push audio to queue\n"); + return MRCP_STATUS_FAILURE; + } + + return MRCP_STATUS_SUCCESS; + +} + +long add_up_buffer(switch_byte_t *buffer, size_t len) { + // add up contents in buffer + long result = 0; + for (int i=0; i<len; i++) { + result =+ buffer[i]; + } + return result; +} + /** - * Called back by openmcp client thread every time its ready for more audio - * Reads data that was put into a shared fifo queue upon receiving audio frames - * from asr_feed() + * Called back by openmcp client thread every time its ready for more audio to send + * the recognition server we are connected to. Reads data that was put into a + * shared fifo queue upon receiving audio frames from asr_feed() */ static apr_status_t openmrcp_recognizer_read_frame(audio_source_t *source, media_frame_t *frame) { - openmrcp_session_t *openmrcp_session = source->object; + asr_session_t *asr_session = source->object; frame->type = MEDIA_FRAME_TYPE_NONE; apr_status_t result; media_frame_t *queue_frame = NULL; - /* pop next media frame data from incoming queue into frame */ - if(openmrcp_session->audio_input_queue) { - if (switch_queue_trypop(openmrcp_session->audio_input_queue, (void *) &queue_frame)) { + if(asr_session->audio_queue) { + if (switch_queue_trypop(asr_session->audio_queue, (void *) &queue_frame)) { // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "could not pop from queue\n"); result = MRCP_STATUS_FAILURE; } @@ -383,7 +596,7 @@ } } else { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "no audio_in queue\n"); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "no audio queue\n"); result = MRCP_STATUS_FAILURE; } @@ -392,7 +605,7 @@ /** Read in the grammar and construct an MRCP Recognize message that has The grammar attached as the payload */ -static mrcp_status_t openmrcp_recog_start(mrcp_client_context_t *context, openmrcp_session_t *openmrcp_session, char *path) +static mrcp_status_t openmrcp_recog_start(mrcp_client_context_t *context, asr_session_t *asr_session, char *path) { mrcp_generic_header_t *generic_header; @@ -403,7 +616,7 @@ char *buf1; apr_size_t bytes2read = 0; - mrcp_message_t *mrcp_message = mrcp_client_context_message_get(context, openmrcp_session->client_session, openmrcp_session->channel, RECOGNIZER_RECOGNIZE); + mrcp_message_t *mrcp_message = mrcp_client_context_message_get(context, asr_session->client_session, asr_session->channel, RECOGNIZER_RECOGNIZE); if(!mrcp_message) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not create mrcp msg\n"); @@ -431,7 +644,7 @@ mrcp_message->body = mrcp_str_pdup(mrcp_message->pool,buf1); /* send the MRCP RECOGNIZE message to MRCP server */ - return mrcp_client_context_channel_modify(context, openmrcp_session->client_session, mrcp_message); + return mrcp_client_context_channel_modify(context, asr_session->client_session, mrcp_message); } @@ -446,8 +659,8 @@ */ static switch_status_t openmrcp_asr_open(switch_asr_handle_t *ah, char *codec, int rate, char *dest, switch_asr_flag_t *flags) { - mrcp_client_context_t *mrcp_client_context = openmrcp_module.mrcp_client_context ; - openmrcp_session_t *openmrcp_session; + mrcp_client_context_t *asr_client_context = openmrcp_module.asr_client_context ; + asr_session_t *asr_session; audio_source_t *source; switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "asr_open called, codec: %s, rate: %d\n", codec, rate); @@ -483,37 +696,37 @@ } /* create session */ - openmrcp_session = openmrcp_session_create(); - if (!openmrcp_session) { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "openmrcp_session creation FAILED\n"); + asr_session = asr_session_create(); + if (!asr_session) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "asr_session creation FAILED\n"); return SWITCH_STATUS_GENERR; } - openmrcp_session->client_session = mrcp_client_context_session_create(mrcp_client_context,openmrcp_session); - if (!openmrcp_session->client_session) { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "openmrcp_session creation FAILED\n"); + asr_session->client_session = mrcp_client_context_session_create(asr_client_context,asr_session); + if (!asr_session->client_session) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "asr_session creation FAILED\n"); return SWITCH_STATUS_GENERR; } /* create audio source */ - source = mrcp_palloc(openmrcp_session->pool,sizeof(audio_source_t)); + source = mrcp_palloc(asr_session->pool,sizeof(audio_source_t)); source->method_set = &audio_source_method_set; - source->object = openmrcp_session; - openmrcp_session->source = source; + source->object = asr_session; + asr_session->source = source; /** * create a new fifo queue. incoming audio received from freeswitch * will be put into this queue, and it will be later pulled out by * the openmrcp client thread. */ - if (switch_queue_create(&openmrcp_session->audio_input_queue, 10000, openmrcp_session->pool)) { + if (switch_queue_create(&asr_session->audio_queue, 10000, asr_session->pool)) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "audio queue creation failed\n"); return SWITCH_STATUS_MEMERR; } - openmrcp_session->flags = *flags; - switch_mutex_init(&openmrcp_session->flag_mutex, SWITCH_MUTEX_NESTED, openmrcp_session->pool); + asr_session->flags = *flags; + switch_mutex_init(&asr_session->flag_mutex, SWITCH_MUTEX_NESTED, asr_session->pool); - ah->private_info = openmrcp_session; + ah->private_info = asr_session; return SWITCH_STATUS_SUCCESS; @@ -528,28 +741,28 @@ TODO: - how does DEFINE-GRAMMAR fit into the picture here? (if at all) */ - openmrcp_session_t *openmrcp_session = (openmrcp_session_t *) ah->private_info; - mrcp_client_context_t *mrcp_client_context = openmrcp_module.mrcp_client_context; - audio_source_t *source = openmrcp_session->source; + asr_session_t *asr_session = (asr_session_t *) ah->private_info; + mrcp_client_context_t *asr_client_context = openmrcp_module.asr_client_context; + audio_source_t *source = asr_session->source; /* create recognizer channel, also starts outgoing rtp media */ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "Loading grammar\n"); switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Create Recognizer Channel\n"); - openmrcp_session->channel = mrcp_client_recognizer_channel_create(mrcp_client_context, openmrcp_session->client_session, source); + asr_session->channel = mrcp_client_recognizer_channel_create(asr_client_context, asr_session->client_session, source); - if (!openmrcp_session->channel) { + if (!asr_session->channel) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Failed to create recognizer channel\n"); return SWITCH_STATUS_FALSE; } /* wait for recognizer channel creation */ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "WAITING FOR CHAN CREATE\n"); - if(openmrcp_session_wait_for_event(openmrcp_session, OPENMRCP_EVENT_CHANNEL_CREATE, OPENMRCP_WAIT_TIMEOUT) == MRCP_STATUS_SUCCESS) { + if(wait_for_event(asr_session->event_queue, OPENMRCP_EVENT_CHANNEL_CREATE, OPENMRCP_WAIT_TIMEOUT) == MRCP_STATUS_SUCCESS) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Got channel creation event\n"); switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "WAITING FOR NONE\n"); - if (openmrcp_session_wait_for_event(openmrcp_session,OPENMRCP_EVENT_NONE,200) == MRCP_STATUS_SUCCESS) { + if (wait_for_event(asr_session->event_queue,OPENMRCP_EVENT_NONE,200) == MRCP_STATUS_SUCCESS) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "GOT NONE EVENT\n"); } else { @@ -557,7 +770,7 @@ } switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Start Recognizer\n"); - openmrcp_recog_start(mrcp_client_context, openmrcp_session, path); + openmrcp_recog_start(asr_client_context, asr_session, path); } else { @@ -573,12 +786,12 @@ static switch_status_t openmrcp_asr_feed(switch_asr_handle_t *ah, void *data, unsigned int len, switch_asr_flag_t *flags) { - openmrcp_session_t *openmrcp_session = (openmrcp_session_t *) ah->private_info; + asr_session_t *asr_session = (asr_session_t *) ah->private_info; media_frame_t *media_frame; switch_byte_t *buffer; /* create new media frame */ - media_frame = (media_frame_t *) switch_core_alloc(openmrcp_session->pool, sizeof(media_frame_t)); + media_frame = (media_frame_t *) switch_core_alloc(asr_session->pool, sizeof(media_frame_t)); if (!media_frame) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "media_frame creation failed\n"); return SWITCH_STATUS_MEMERR; @@ -592,7 +805,7 @@ * this needs to send the packet directly. OpenMrcp will need a way to disable their own * timer so that the fs timer is the only one driving the media. **/ - buffer = (switch_byte_t *) switch_core_alloc(openmrcp_session->pool, sizeof(switch_byte_t)*len); + buffer = (switch_byte_t *) switch_core_alloc(asr_session->pool, sizeof(switch_byte_t)*len); if (!buffer) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not allocate buffer\n"); return SWITCH_STATUS_MEMERR; @@ -604,7 +817,7 @@ media_frame->type = MEDIA_FRAME_TYPE_AUDIO; /* push audio to queue */ - if (switch_queue_trypush(openmrcp_session->audio_input_queue, (void *) media_frame)) { + if (switch_queue_trypush(asr_session->audio_queue, (void *) media_frame)) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "could not push audio to queue\n"); return SWITCH_STATUS_GENERR; } @@ -644,19 +857,19 @@ */ static switch_status_t openmrcp_asr_close(switch_asr_handle_t *ah, switch_asr_flag_t *flags) { - openmrcp_session_t *openmrcp_session = (openmrcp_session_t *) ah->private_info; + asr_session_t *asr_session = (asr_session_t *) ah->private_info; switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "openmrcp_asr_close()\n"); - mrcp_client_context_t *context = openmrcp_module.mrcp_client_context; + mrcp_client_context_t *context = openmrcp_module.asr_client_context; // TODO!! should we do a switch_pool_clear(switch_memory_pool_t *p) on the pool held - // by openmrcp_session? + // by asr_session? // destroy channel switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Going to DESTROY CHANNEL\n"); - mrcp_client_context_channel_destroy(context, openmrcp_session->client_session, openmrcp_session->channel); - if (openmrcp_session_wait_for_event(openmrcp_session,OPENMRCP_EVENT_CHANNEL_DESTROY,10000) == MRCP_STATUS_SUCCESS) { + mrcp_client_context_channel_destroy(context, asr_session->client_session, asr_session->channel); + if (wait_for_event(asr_session->event_queue,OPENMRCP_EVENT_CHANNEL_DESTROY,10000) == MRCP_STATUS_SUCCESS) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "OPENMRCP_EVENT_CHANNEL_DESTROY received\n"); } else { @@ -665,8 +878,8 @@ // terminate client session switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Going to TERMINATE SESSION\n"); - mrcp_client_context_session_terminate(context, openmrcp_session->client_session); - if (openmrcp_session_wait_for_event(openmrcp_session,OPENMRCP_EVENT_SESSION_TERMINATE,10000) == MRCP_STATUS_SUCCESS) { + mrcp_client_context_session_terminate(context, asr_session->client_session); + if (wait_for_event(asr_session->event_queue,OPENMRCP_EVENT_SESSION_TERMINATE,10000) == MRCP_STATUS_SUCCESS) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "OPENMRCP_EVENT_SESSION_TERMINATE recevied\n"); } else { @@ -675,10 +888,10 @@ // destroy client session (NOTE: this sends a BYE to the other side) switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Going to DESTROY SESSION\n"); - mrcp_client_context_session_destroy(context, openmrcp_session->client_session); + mrcp_client_context_session_destroy(context, asr_session->client_session); - // destroys the openmrcp_session struct - openmrcp_session_destroy(openmrcp_session); + // destroys the asr_session struct + asr_session_destroy(asr_session); switch_set_flag(ah, SWITCH_ASR_FLAG_CLOSED); @@ -694,9 +907,9 @@ */ static switch_status_t openmrcp_asr_check_results(switch_asr_handle_t *ah, switch_asr_flag_t *flags) { - openmrcp_session_t *openmrcp_session = (openmrcp_session_t *) ah->private_info; + asr_session_t *asr_session = (asr_session_t *) ah->private_info; - switch_status_t rv = (switch_test_flag(openmrcp_session, FLAG_HAS_TEXT) || switch_test_flag(openmrcp_session, FLAG_BARGE)) ? SWITCH_STATUS_SUCCESS : SWITCH_STATUS_FALSE; + switch_status_t rv = (switch_test_flag(asr_session, FLAG_HAS_TEXT) || switch_test_flag(asr_session, FLAG_BARGE)) ? SWITCH_STATUS_SUCCESS : SWITCH_STATUS_FALSE; return rv; } @@ -704,16 +917,16 @@ /*! This will be called after asr_check_results returns SUCCESS */ static switch_status_t openmrcp_asr_get_results(switch_asr_handle_t *ah, char **xmlstr, switch_asr_flag_t *flags) { - openmrcp_session_t *openmrcp_session = (openmrcp_session_t *) ah->private_info; + asr_session_t *asr_session = (asr_session_t *) ah->private_info; switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "openmrcp_asr_get_results called\n"); switch_status_t ret = SWITCH_STATUS_SUCCESS; - if (switch_test_flag(openmrcp_session, FLAG_BARGE)) { - switch_clear_flag_locked(openmrcp_session, FLAG_BARGE); + if (switch_test_flag(asr_session, FLAG_BARGE)) { + switch_clear_flag_locked(asr_session, FLAG_BARGE); ret = SWITCH_STATUS_BREAK; } - if (switch_test_flag(openmrcp_session, FLAG_HAS_TEXT)) { + if (switch_test_flag(asr_session, FLAG_HAS_TEXT)) { /*! we have to extract the XML but stripping off the <?xml version="1.0"?> header. the body looks like: @@ -724,16 +937,15 @@ <?xml version="1.0"?> <result xmlns="http://www.ietf.org/xml/ns/mrcpv2" xmlns:ex="http://www.example.com/example" score="100" grammar="session:request1@form-level.store"> - <interpretation> - <input mode="speech">open a</input> + <interpretation> <input mode="speech">open a</input> </interpretation> </result> */ char *marker = "?>"; // FIXME -- lame and brittle way of doing this. use regex or better. - char *position = strstr(openmrcp_session->mrcp_message_last_rcvd->body, marker); + char *position = strstr(asr_session->mrcp_message_last_rcvd->body, marker); if (!position) { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Bad result received from mrcp server: %s", openmrcp_session->mrcp_message_last_rcvd->body); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Bad result received from mrcp server: %s", asr_session->mrcp_message_last_rcvd->body); ret = SWITCH_STATUS_FALSE; } else { @@ -743,12 +955,236 @@ // since we are returning our result here, future calls to check_results // should return False - switch_clear_flag_locked(openmrcp_session, FLAG_HAS_TEXT); + switch_clear_flag_locked(asr_session, FLAG_HAS_TEXT); ret = SWITCH_STATUS_SUCCESS; } return ret; } + +static mrcp_status_t synth_speak(mrcp_client_context_t *context, tts_session_t *tts_session, char *text) +{ + + //buffer = (switch_byte_t *) switch_core_alloc(asr_session->pool, sizeof(switch_byte_t)*len); + char *text2speak; + const char xml_head[] = + "<?xml version=\"1.0\"?>\r\n" + "<speak>\r\n" + "<paragraph>\r\n" + " <sentence>"; + + const char xml_tail[] = "</sentence>\r\n" + "</paragraph>\r\n" + "</speak>\r\n"; + + size_t len = sizeof(xml_head) + sizeof(text) + sizeof(xml_tail); + text2speak = (char *) switch_core_alloc(tts_session->pool, len); + strcat(text2speak, xml_head); + strcat(text2speak, text); + strcat(text2speak, xml_tail); + + mrcp_generic_header_t *generic_header; + mrcp_message_t *mrcp_message = mrcp_client_context_message_get(context,tts_session->client_session,tts_session->channel,SYNTHESIZER_SPEAK); + if(!mrcp_message) { + return MRCP_STATUS_FAILURE; + } + + generic_header = mrcp_generic_header_prepare(mrcp_message); + if(!generic_header) { + return MRCP_STATUS_FAILURE; + } + + generic_header->content_type = mrcp_str_pdup(mrcp_message->pool,"application/synthesis+ssml"); + mrcp_generic_header_property_add(mrcp_message,GENERIC_HEADER_CONTENT_TYPE); + mrcp_message->body = mrcp_str_pdup(mrcp_message->pool,text2speak); + + return mrcp_client_context_channel_modify(context,tts_session->client_session,mrcp_message); +} + +static mrcp_status_t synth_stop(mrcp_client_context_t *context, tts_session_t *tts_session) +{ + mrcp_message_t *mrcp_message = mrcp_client_context_message_get(context,tts_session->client_session,tts_session->channel,SYNTHESIZER_STOP); + if(!mrcp_message) { + return MRCP_STATUS_FAILURE; + } + + return mrcp_client_context_channel_modify(context,tts_session->client_session,mrcp_message); +} + + +static switch_status_t openmrcp_tts_open(switch_speech_handle_t *sh, char *voice_name, int rate, switch_speech_flag_t *flags) +{ + + tts_session_t *tts_session; + audio_sink_t *sink; + mrcp_client_context_t *tts_client_context = openmrcp_module.tts_client_context ; + + /* create session */ + tts_session = tts_session_create(); + if (!tts_session) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "tts_session creation FAILED\n"); + return SWITCH_STATUS_GENERR; + } + tts_session->client_session = mrcp_client_context_session_create(tts_client_context,tts_session); + if (!tts_session->client_session) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "tts_session creation FAILED\n"); + return SWITCH_STATUS_GENERR; + } + + + /* create audio sink */ + sink = mrcp_palloc(tts_session->pool,sizeof(audio_sink_t)); + sink->method_set = &audio_sink_method_set; + sink->object = tts_session; + tts_session->sink = sink; + + + /** + * create a new fifo queue. + */ + if (switch_queue_create(&tts_session->audio_queue, 10000, tts_session->pool)) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "audio queue creation failed\n"); + return SWITCH_STATUS_MEMERR; + } + + + tts_session->flags = *flags; + switch_mutex_init(&tts_session->flag_mutex, SWITCH_MUTEX_NESTED, tts_session->pool); + + sh->private_info = tts_session; + + return SWITCH_STATUS_SUCCESS; + +} + +static switch_status_t openmrcp_tts_close(switch_speech_handle_t *sh, switch_speech_flag_t *flags) +{ + tts_session_t *tts_session = (tts_session_t *) sh->private_info; + mrcp_client_context_t *tts_client_context = openmrcp_module.tts_client_context ; + + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "synth_stop\n"); + synth_stop(tts_client_context,tts_session); // TODO + wait_for_event(tts_session->event_queue,OPENMRCP_EVENT_CHANNEL_MODIFY,5000); + + /* terminate tts session */ + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "Terminate tts_session\n"); + mrcp_client_context_session_terminate(tts_client_context,tts_session->client_session); + /* wait for tts session termination */ + wait_for_event(tts_session->event_queue,OPENMRCP_EVENT_SESSION_TERMINATE,10000); + + /* destroy demo session */ + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "destroy tts_session\n"); + mrcp_client_context_session_destroy(tts_client_context,tts_session->client_session); + tts_session_destroy(tts_session); + + return SWITCH_STATUS_SUCCESS; +} + +static switch_status_t openmrcp_feed_tts(switch_speech_handle_t *sh, char *text, switch_speech_flag_t *flags) +{ + + tts_session_t *tts_session = (tts_session_t *) sh->private_info; + mrcp_client_context_t *tts_client_context = openmrcp_module.tts_client_context ; + audio_sink_t *sink = tts_session->sink; + + /* create synthesizer channel */ + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "Create Synthesizer Channel\n"); + tts_session->channel = mrcp_client_synthesizer_channel_create(tts_client_context,tts_session->client_session,sink); + /* wait for synthesizer channel creation */ + if(wait_for_event(tts_session->event_queue,OPENMRCP_EVENT_CHANNEL_CREATE,5000) == MRCP_STATUS_SUCCESS) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "Got channel create event\n"); + wait_for_event(tts_session->event_queue,OPENMRCP_EVENT_NONE,1000); + /* speak */ + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "Going to speak\n"); + synth_speak(tts_client_context,tts_session, text); + // wait_for_event(tts_session->event_queue,OPENMRCP_EVENT_CHANNEL_MODIFY,5000); + + } + else { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Never got channel create event.\n"); + return SWITCH_STATUS_FALSE; + } + + + + return SWITCH_STATUS_SUCCESS; +} + +static void openmrcp_flush_tts(switch_speech_handle_t *sh) +{ + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "flush_tts called\n"); +} + +/** + * Freeswitch calls this when its ready to read datalen bytes of data. + * + * TODO: check the blocking flag and act accordingly + */ +static switch_status_t openmrcp_read_tts(switch_speech_handle_t *sh, void *data, size_t *datalen, uint32_t *rate, switch_speech_flag_t *flags) +{ + media_frame_t *queue_frame = NULL; + tts_session_t *tts_session = (tts_session_t *) sh->private_info; + size_t len = *datalen; + size_t x = 0; + + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "openmrcp_read_tts, datalen: %d\n", len); + + if (switch_queue_trypop(tts_session->audio_queue, (void *) &queue_frame)) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "could not pop from queue\n"); + //return SWITCH_STATUS_FALSE; + return SWITCH_STATUS_SUCCESS; + } + else { + + if (queue_frame->codec_frame.size != len) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "You must recompile openmrcp with #define CODEC_FRAME_TIME_BASE 20\n"); + return SWITCH_STATUS_FALSE; + } + + + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "add_up queue_frame: %li\n", + add_up_buffer(queue_frame->codec_frame.buffer, len)); + + + + switch_byte_t *bytedata = (switch_byte_t *) data; + switch_byte_t *sourcedata = (switch_byte_t *) queue_frame->codec_frame.buffer; + + // memcpy(data, queue_frame->codec_frame.buffer, len); + for (int i=0; i<len; i++) { + *(bytedata + i) = sourcedata[i]; + } + + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "add_up bytedata: %li\n", + add_up_buffer(bytedata, len)); + + + + // assume openmrcp is using 8kz for now + *rate = 8000; + + return SWITCH_STATUS_SUCCESS; + } + + + return SWITCH_STATUS_FALSE; +} + +static void openmrcp_text_param_tts(switch_speech_handle_t *sh, char *param, char *val) +{ + +} + +static void openmrcp_numeric_param_tts(switch_speech_handle_t *sh, char *param, int val) +{ + +} + +static void openmrcp_float_param_tts(switch_speech_handle_t *sh, char *param, double val) +{ + +} + static switch_asr_interface_t openmrcp_asr_interface = { /*.interface_name*/ "openmrcp", /*.asr_open*/ openmrcp_asr_open, @@ -762,6 +1198,19 @@ /*.asr_get_results*/ openmrcp_asr_get_results }; +static switch_speech_interface_t openmrcp_tts_interface = { + /*.interface_name*/ "openmrcp", + /*.speech_open*/ openmrcp_tts_open, + /*.speech_close*/ openmrcp_tts_close, + /*.speech_feed_tts*/ openmrcp_feed_tts, + /*.speech_read_tts*/ openmrcp_read_tts, + /*.speech_flush_tts*/ openmrcp_flush_tts, + /*.speech_text_param_tts*/ openmrcp_text_param_tts, + /*.speech_numeric_param_tts*/ openmrcp_numeric_param_tts, + /*.speech_float_param_tts*/ openmrcp_float_param_tts, + +}; + static switch_loadable_module_interface_t openmrcp_module_interface = { /*.module_name */ modname, /*.endpoint_interface */ NULL, @@ -771,7 +1220,7 @@ /*.application_interface */ NULL, /*.api_interface */ NULL, /*.file_interface */ NULL, - /*.speech_interface */ NULL, + /*.speech_interface */ &openmrcp_tts_interface, /*.directory_interface */ NULL, /*.chat_interface */ NULL, /*.say_interface */ NULL, @@ -796,16 +1245,26 @@ char *var = (char *) switch_xml_attr_soft(param, "name"); char *val = (char *) switch_xml_attr_soft(param, "value"); - if (!strcasecmp(var, "client_ip")) { - globals.client_ip = val; - } else if (!strcasecmp(var, "server_ip")) { - globals.server_ip = val; - } else if (!strcasecmp(var, "proto_version")) { - globals.proto_version =(uint32_t) atoi(val); - } else if (!strcasecmp(var, "client_port")) { - globals.client_port = (uint32_t) atoi(val); - } else if (!strcasecmp(var, "server_port")) { - globals.server_port = (uint32_t) atoi(val); + if (!strcasecmp(var, "asr_client_ip")) { + globals.asr_client_ip = val; + } else if (!strcasecmp(var, "asr_server_ip")) { + globals.asr_server_ip = val; + } else if (!strcasecmp(var, "asr_proto_version")) { + globals.asr_proto_version =(uint32_t) atoi(val); + } else if (!strcasecmp(var, "asr_client_port")) { + globals.asr_client_port = (uint32_t) atoi(val); + } else if (!strcasecmp(var, "asr_server_port")) { + globals.asr_server_port = (uint32_t) atoi(val); + } else if (!strcasecmp(var, "tts_client_ip")) { + globals.tts_client_ip = val; + } else if (!strcasecmp(var, "tts_server_ip")) { + globals.tts_server_ip = val; + } else if (!strcasecmp(var, "tts_proto_version")) { + globals.tts_proto_version =(uint32_t) atoi(val); + } else if (!strcasecmp(var, "tts_client_port")) { + globals.tts_client_port = (uint32_t) atoi(val); + } else if (!strcasecmp(var, "tts_server_port")) { + globals.tts_server_port = (uint32_t) atoi(val); } else if (!strcasecmp(var, "rtp_port_min")) { globals.rtp_port_min = (uint32_t) atoi(val); } else if (!strcasecmp(var, "rtp_port_max")) { @@ -822,44 +1281,86 @@ static switch_status_t mrcp_init() { /*! - Perform one-time initialization of client library + Perform one-time initialization of asr client library */ mrcp_mem_pool_t *pool; - mrcp_client_event_handler_t *client_event_handler; - mrcp_client_t *mrcp_client; - mrcp_client_context_t *mrcp_client_context; - openmrcp_client_options_t *options; + mrcp_client_event_handler_t *asr_event_handler; + mrcp_client_t *asr_client; + mrcp_client_context_t *asr_client_context; + openmrcp_client_options_t *asr_options; + + pool = mrcp_global_pool_get(); + asr_options = mrcp_palloc(pool,sizeof(openmrcp_client_options_t)); + asr_event_handler = mrcp_palloc(pool,sizeof(mrcp_client_event_handler_t)); + set_default_asr_options(asr_options); + + asr_event_handler->on_session_initiate = asr_on_session_initiate; + asr_event_handler->on_session_terminate = asr_on_session_terminate; + asr_event_handler->on_channel_add = asr_on_channel_add; + asr_event_handler->on_channel_remove = asr_on_channel_remove; + asr_event_handler->on_channel_modify = asr_on_channel_modify; + + // create asr client context, which to must be passed to client engine + asr_client_context = mrcp_client_context_create(&openmrcp_module,asr_event_handler); + if(!asr_client_context) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "asr_client_context creation failed\n"); + return SWITCH_STATUS_GENERR; + } + openmrcp_module.asr_client_context = asr_client_context; + + // this basically starts a thread that pulls events from the event queue + // and handles them + asr_client = openmrcp_client_start(asr_options,asr_client_context); + if(!asr_client) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "openasr_client_start FAILED\n"); + mrcp_client_context_destroy(asr_client_context); + return SWITCH_STATUS_GENERR; + } + + openmrcp_module.asr_client = asr_client; + + + /*! + Perform one-time initialization of tts client library + */ + + mrcp_client_event_handler_t *tts_event_handler; + mrcp_client_t *tts_client; + mrcp_client_context_t *tts_client_context; + openmrcp_client_options_t *tts_options; pool = mrcp_global_pool_get(); - options = mrcp_palloc(pool,sizeof(openmrcp_client_options_t)); - client_event_handler = mrcp_palloc(pool,sizeof(mrcp_client_event_handler_t)); - set_default_options(options); - - client_event_handler->on_session_initiate = openmrcp_on_session_initiate; - client_event_handler->on_session_terminate = openmrcp_on_session_terminate; - client_event_handler->on_channel_add = openmrcp_on_channel_add; - client_event_handler->on_channel_remove = openmrcp_on_channel_remove; - client_event_handler->on_channel_modify = openmrcp_on_channel_modify; - - // create client context, which to must be passed to client engine - mrcp_client_context = mrcp_client_context_create(&openmrcp_module,client_event_handler); - if(!mrcp_client_context) { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "mrcp_client_context creation failed\n"); + tts_options = mrcp_palloc(pool,sizeof(openmrcp_client_options_t)); + tts_event_handler = mrcp_palloc(pool,sizeof(mrcp_client_event_handler_t)); + set_default_tts_options(tts_options); + + tts_event_handler->on_session_initiate = tts_on_session_initiate; + tts_event_handler->on_session_terminate = tts_on_session_terminate; + tts_event_handler->on_channel_add = tts_on_channel_add; + tts_event_handler->on_channel_remove = tts_on_channel_remove; + tts_event_handler->on_channel_modify = tts_on_channel_modify; + + // create tts client context, which to must be passed to client engine + tts_client_context = mrcp_client_context_create(&openmrcp_module,tts_event_handler); + if(!tts_client_context) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "tts_client_context creation failed\n"); return SWITCH_STATUS_GENERR; } - openmrcp_module.mrcp_client_context = mrcp_client_context; + openmrcp_module.tts_client_context = tts_client_context; // this basically starts a thread that pulls events from the event queue // and handles them - mrcp_client = openmrcp_client_start(options,mrcp_client_context); - if(!mrcp_client) { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "openmrcp_client_start FAILED\n"); - mrcp_client_context_destroy(mrcp_client_context); + tts_client = openmrcp_client_start(tts_options,tts_client_context); + if(!tts_client) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "opentts_client_start FAILED\n"); + mrcp_client_context_destroy(tts_client_context); return SWITCH_STATUS_GENERR; } - openmrcp_module.mrcp_client = mrcp_client; + openmrcp_module.tts_client = tts_client; + + return SWITCH_STATUS_SUCCESS; }

1 0

[Freeswitch-branches] [commit] r5587 - freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp
by Freeswitch SVN 11 Aug '07

11 Aug '07

Author: greenlizard Date: Sat Aug 11 12:28:24 2007 New Revision: 5587 Modified: freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/Makefile freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/mod_openmrcp.c Log: works with trunk version of openmrcp since it now sets the version correctly Modified: freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/Makefile ============================================================================== --- freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/Makefile (original) +++ freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/Makefile Sat Aug 11 12:28:24 2007 @@ -14,9 +14,7 @@ # - is there an openmrcp dir in the fs tree yet? (see FIXME) # FIXME!!! Should this download and build openmrcp? What's the plan? -# NOTE: currently only works with trunk revision 367. On trunk revision 440 -# there is a segfault when server loads. -OPENMRCP_DIR=/usr/src/openmrcp_stable +OPENMRCP_DIR=/usr/src/openmrcp_trunk OPENMRCP_INCLUDE=-I$(OPENMRCP_DIR)/mrcpcore/client/include/ -I$(OPENMRCP_DIR)/platform/openmrcpclient/include/ -I$(OPENMRCP_DIR)/mediaframe/include/ -I$(OPENMRCP_DIR)/mrcpcore/engine/include/ -I$(OPENMRCP_DIR)/mrcpcore/include/ -I$(OPENMRCP_DIR)/mrcpcore/parser/include/ -I$(OPENMRCP_DIR)/mrcpcore/server/include/ -I$(OPENMRCP_DIR)/mrcpcore/media/include/ -I$(OPENMRCP_DIR)/mrcpcore/util/include -I$(OPENMRCP_DIR)/mrcpcore/resource/include/ @@ -24,7 +22,7 @@ OPENMRCP_LIBS_STABLE=$(OPENMRCP_DIR)/platform/openmrcpclient/.libs/libopenmrcpclient.a $(OPENMRCP_DIR)/platform/openmrcpserver/.libs/libopenmrcpserver.a $(OPENMRCP_DIR)/mrcpcore/client/.libs/libmrcpclient.a $(OPENMRCP_DIR)/mediaframe/.libs/libmediaframe.a $(OPENMRCP_DIR)/mrcpcore/engine/.libs/libmrcpengine.a $(OPENMRCP_DIR)/mrcpcore/parser/.libs/libmrcpparser.a $(OPENMRCP_DIR)/mrcpcore/server/.libs/libmrcpserver.a $(OPENMRCP_DIR)/mrcpcore/media/.libs/libmrcpmedia.a $(OPENMRCP_DIR)/mrcpcore/util/.libs/libmrcputil.a $(OPENMRCP_DIR)/mrcpcore/util/.libs/libmrcputil.a $(OPENMRCP_DIR)/mrcpcore/resource/.libs/libmrcpresource.a $(OPENMRCP_DIR)/module/plugin/demo/.libs/libdemoplugin.a $(OPENMRCP_DIR)/module/mrcpv2agent/.libs/libmrcpv2agent.a $(OPENMRCP_DIR)/module/mrcplivemedia/.libs/libmrcplivemedia.a $(OPENMRCP_DIR)/module/mrcpsofiasip/.libs/libmrcpsofiasip.a $(OPENMRCP_DIR)/mrcpcore/.libs/libmrcpcore.a -OPENMRCP_LIBS=$(OPENMRCP_LIBS_STABLE) +OPENMRCP_LIBS=$(OPENMRCP_LIBS_TRUNK) MRCP_APR_INCLUDES = -I../../../../libs/apr/include MRCP_APU_INCLUDES = -I../../../../libs/apr-util/include MRCP_SOFIA_INCLUDES = -I/usr/local/include/sofia-sip-1.12 # FIXME!! Modified: freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/mod_openmrcp.c ============================================================================== --- freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/mod_openmrcp.c (original) +++ freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/mod_openmrcp.c Sat Aug 11 12:28:24 2007 @@ -27,10 +27,11 @@ * * Module which acts as an MRCP client to an MRCP speech recognition * server. In other words it bridges freeswitch to an external speech - * recognition system. + * recognition system. Documentation on how to install and configure + * the module is here: http://wiki.freeswitch.org/wiki/Mod_openmrcp * * Uses OpenMrcp (http://wiki.freeswitch.org/wiki/OpenMRCP) as the - * the client library. + * the client library. * */ #ifdef __ICC @@ -70,6 +71,7 @@ static struct { char *client_ip; char *server_ip; + uint32_t proto_version; uint32_t client_port; uint32_t server_port; uint32_t rtp_port_min; @@ -197,7 +199,6 @@ timeout = 0; } else { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "sleeping %d millieconds\n", sleep_ms); switch_sleep(sleep_ms * 1000); // ms->us timeout -= sleep_ms; } @@ -344,6 +345,7 @@ static apr_status_t set_default_options(openmrcp_client_options_t *options) { mrcp_logger.priority = MRCP_PRIO_INFO; + options->proto_version = globals.proto_version; options->client_ip = globals.client_ip; options->server_ip = globals.server_ip; options->client_port = globals.client_port; @@ -798,6 +800,8 @@ globals.client_ip = val; } else if (!strcasecmp(var, "server_ip")) { globals.server_ip = val; + } else if (!strcasecmp(var, "proto_version")) { + globals.proto_version =(uint32_t) atoi(val); } else if (!strcasecmp(var, "client_port")) { globals.client_port = (uint32_t) atoi(val); } else if (!strcasecmp(var, "server_port")) {

1 0

[Freeswitch-branches] [commit] r5586 - freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp
by Freeswitch SVN 11 Aug '07

11 Aug '07

Author: greenlizard Date: Fri Aug 10 21:56:34 2007 New Revision: 5586 Modified: freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/Makefile freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/mod_openmrcp.c Log: removed all compiler warnings emanating from mod_openmrcp.c, though there are still a few remaining that are coming from the library Modified: freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/Makefile ============================================================================== --- freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/Makefile (original) +++ freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/Makefile Fri Aug 10 21:56:34 2007 @@ -13,13 +13,18 @@ # - needs to compile/link against the _freeswitch_ sofia libs (see FIXME) # - is there an openmrcp dir in the fs tree yet? (see FIXME) -# FIXME!!! +# FIXME!!! Should this download and build openmrcp? What's the plan? +# NOTE: currently only works with trunk revision 367. On trunk revision 440 +# there is a segfault when server loads. OPENMRCP_DIR=/usr/src/openmrcp_stable OPENMRCP_INCLUDE=-I$(OPENMRCP_DIR)/mrcpcore/client/include/ -I$(OPENMRCP_DIR)/platform/openmrcpclient/include/ -I$(OPENMRCP_DIR)/mediaframe/include/ -I$(OPENMRCP_DIR)/mrcpcore/engine/include/ -I$(OPENMRCP_DIR)/mrcpcore/include/ -I$(OPENMRCP_DIR)/mrcpcore/parser/include/ -I$(OPENMRCP_DIR)/mrcpcore/server/include/ -I$(OPENMRCP_DIR)/mrcpcore/media/include/ -I$(OPENMRCP_DIR)/mrcpcore/util/include -I$(OPENMRCP_DIR)/mrcpcore/resource/include/ -OPENMRCP_LIBS=$(OPENMRCP_DIR)/platform/openmrcpclient/.libs/libopenmrcpclient.a $(OPENMRCP_DIR)/platform/openmrcpserver/.libs/libopenmrcpserver.a $(OPENMRCP_DIR)/mrcpcore/client/.libs/libmrcpclient.a $(OPENMRCP_DIR)/mediaframe/.libs/libmediaframe.a $(OPENMRCP_DIR)/mrcpcore/engine/.libs/libmrcpengine.a $(OPENMRCP_DIR)/mrcpcore/parser/.libs/libmrcpparser.a $(OPENMRCP_DIR)/mrcpcore/server/.libs/libmrcpserver.a $(OPENMRCP_DIR)/mrcpcore/media/.libs/libmrcpmedia.a $(OPENMRCP_DIR)/mrcpcore/util/.libs/libmrcputil.a $(OPENMRCP_DIR)/mrcpcore/util/.libs/libmrcputil.a $(OPENMRCP_DIR)/mrcpcore/resource/.libs/libmrcpresource.a $(OPENMRCP_DIR)/module/plugin/demo/.libs/libdemoplugin.a $(OPENMRCP_DIR)/module/mrcpv2agent/.libs/libmrcpv2agent.a $(OPENMRCP_DIR)/module/mrcplivemedia/.libs/libmrcplivemedia.a $(OPENMRCP_DIR)/module/mrcpsofiasip/.libs/libmrcpsofiasip.a $(OPENMRCP_DIR)/mrcpcore/.libs/libmrcpcore.a +OPENMRCP_LIBS_TRUNK=$(OPENMRCP_DIR)/platform/openmrcpclient/.libs/libopenmrcpclient.a $(OPENMRCP_DIR)/platform/openmrcpserver/.libs/libopenmrcpserver.a $(OPENMRCP_DIR)/mrcpcore/client/.libs/libmrcpclient.a $(OPENMRCP_DIR)/mediaframe/.libs/libmediaframe.a $(OPENMRCP_DIR)/mrcpcore/engine/.libs/libmrcpengine.a $(OPENMRCP_DIR)/mrcpcore/parser/.libs/libmrcpparser.a $(OPENMRCP_DIR)/mrcpcore/server/.libs/libmrcpserver.a $(OPENMRCP_DIR)/mrcpcore/media/.libs/libmrcpmedia.a $(OPENMRCP_DIR)/mrcpcore/util/.libs/libmrcputil.a $(OPENMRCP_DIR)/mrcpcore/util/.libs/libmrcputil.a $(OPENMRCP_DIR)/mrcpcore/resource/.libs/libmrcpresource.a $(OPENMRCP_DIR)/module/plugin/demo/.libs/libdemoplugin.a $(OPENMRCP_DIR)/module/mrcpv2agent/.libs/libmrcpv2agent.a $(OPENMRCP_DIR)/module/mrcpmediaframe/.libs/libmrcpmediaframe.a $(OPENMRCP_DIR)/module/mrcpsofiasip/.libs/libmrcpsofiasip.a $(OPENMRCP_DIR)/mrcpcore/.libs/libmrcpcore.a +OPENMRCP_LIBS_STABLE=$(OPENMRCP_DIR)/platform/openmrcpclient/.libs/libopenmrcpclient.a $(OPENMRCP_DIR)/platform/openmrcpserver/.libs/libopenmrcpserver.a $(OPENMRCP_DIR)/mrcpcore/client/.libs/libmrcpclient.a $(OPENMRCP_DIR)/mediaframe/.libs/libmediaframe.a $(OPENMRCP_DIR)/mrcpcore/engine/.libs/libmrcpengine.a $(OPENMRCP_DIR)/mrcpcore/parser/.libs/libmrcpparser.a $(OPENMRCP_DIR)/mrcpcore/server/.libs/libmrcpserver.a $(OPENMRCP_DIR)/mrcpcore/media/.libs/libmrcpmedia.a $(OPENMRCP_DIR)/mrcpcore/util/.libs/libmrcputil.a $(OPENMRCP_DIR)/mrcpcore/util/.libs/libmrcputil.a $(OPENMRCP_DIR)/mrcpcore/resource/.libs/libmrcpresource.a $(OPENMRCP_DIR)/module/plugin/demo/.libs/libdemoplugin.a $(OPENMRCP_DIR)/module/mrcpv2agent/.libs/libmrcpv2agent.a $(OPENMRCP_DIR)/module/mrcplivemedia/.libs/libmrcplivemedia.a $(OPENMRCP_DIR)/module/mrcpsofiasip/.libs/libmrcpsofiasip.a $(OPENMRCP_DIR)/mrcpcore/.libs/libmrcpcore.a + +OPENMRCP_LIBS=$(OPENMRCP_LIBS_STABLE) MRCP_APR_INCLUDES = -I../../../../libs/apr/include MRCP_APU_INCLUDES = -I../../../../libs/apr-util/include MRCP_SOFIA_INCLUDES = -I/usr/local/include/sofia-sip-1.12 # FIXME!! Modified: freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/mod_openmrcp.c ============================================================================== --- freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/mod_openmrcp.c (original) +++ freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/mod_openmrcp.c Fri Aug 10 21:56:34 2007 @@ -95,7 +95,7 @@ "CHANNEL_MODIFY", }; -// TODO: rename this to openmrcp_session_t +// TODO: rename this to mod_openmrcp_session_t typedef struct openmrcp_session_t openmrcp_session_t; struct openmrcp_session_t { mrcp_session_t *client_session; @@ -134,22 +134,6 @@ static openmrcp_module_t openmrcp_module; -static void log_callback(const char* message, void* userdata) -{ - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "%s\n", message); -} - -static const char *state_names[] = { - "NOT_READY", - "READY", - "BARGE_IN", - "END_SPEECH", - "STOPPED", - "BARGE_IN_TIMEOUT", - "END_SPEECH_TIMEOUT", - "BEEP" -}; - static openmrcp_session_t* openmrcp_session_create() { openmrcp_session_t *openmrcp_session; @@ -200,13 +184,12 @@ static mrcp_status_t openmrcp_session_wait_for_event(openmrcp_session_t *openmrcp_session, openmrcp_event_t openmrcp_event, size_t timeout) { - mrcp_status_t status = MRCP_STATUS_SUCCESS; + openmrcp_event_t *popped_event = NULL; size_t sleep_ms = 100; - if(openmrcp_session->event_queue) { - if (switch_queue_trypop(openmrcp_session->event_queue, &popped_event)) { + if (switch_queue_trypop(openmrcp_session->event_queue, (void *) &popped_event)) { // most likely this failed because queue is empty. sleep for timeout seconds and try again? if (timeout > 0) { if (timeout < sleep_ms) { @@ -337,9 +320,7 @@ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "!openmrcp_session\n"); return MRCP_STATUS_FAILURE; } - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "openmrcp_session: %p\n", openmrcp_session); - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "mrcp msg method name: %s\n", mrcp_message->start_line.method_name); - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "mrcp msg body: %s\n", mrcp_message->body); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "mrcp msg body: %s\n", mrcp_message->body); if (!strcmp(mrcp_message->start_line.method_name,"RECOGNITION-COMPLETE")) { openmrcp_session->mrcp_message_last_rcvd = mrcp_message; @@ -388,7 +369,7 @@ /* pop next media frame data from incoming queue into frame */ if(openmrcp_session->audio_input_queue) { - if (switch_queue_trypop(openmrcp_session->audio_input_queue, &queue_frame)) { + if (switch_queue_trypop(openmrcp_session->audio_input_queue, (void *) &queue_frame)) { // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "could not pop from queue\n"); result = MRCP_STATUS_FAILURE; } @@ -418,7 +399,7 @@ apr_pool_t *mp; apr_finfo_t finfo; char *buf1; - int bytes2read = 0; + apr_size_t bytes2read = 0; mrcp_message_t *mrcp_message = mrcp_client_context_message_get(context, openmrcp_session->client_session, openmrcp_session->channel, RECOGNIZER_RECOGNIZE); @@ -463,9 +444,7 @@ */ static switch_status_t openmrcp_asr_open(switch_asr_handle_t *ah, char *codec, int rate, char *dest, switch_asr_flag_t *flags) { - mrcp_client_t *mrcp_client = openmrcp_module.mrcp_client; mrcp_client_context_t *mrcp_client_context = openmrcp_module.mrcp_client_context ; - openmrcp_client_options_t *options; openmrcp_session_t *openmrcp_session; audio_source_t *source; @@ -548,7 +527,6 @@ */ openmrcp_session_t *openmrcp_session = (openmrcp_session_t *) ah->private_info; - mrcp_client_t *mrcp_client = openmrcp_module.mrcp_client; mrcp_client_context_t *mrcp_client_context = openmrcp_module.mrcp_client_context; audio_source_t *source = openmrcp_session->source; @@ -594,8 +572,6 @@ { openmrcp_session_t *openmrcp_session = (openmrcp_session_t *) ah->private_info; - mrcp_client_t *mrcp_client = openmrcp_module.mrcp_client; - mrcp_client_context_t *mrcp_client_context = openmrcp_module.mrcp_client_context; media_frame_t *media_frame; switch_byte_t *buffer; @@ -637,7 +613,7 @@ static switch_status_t openmrcp_asr_pause(switch_asr_handle_t *ah) { - openmrcp_session_t *openmrcp_session = (openmrcp_session_t *) ah->private_info; + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "openmrcp_asr_pause called\n"); return SWITCH_STATUS_SUCCESS; @@ -646,7 +622,7 @@ static switch_status_t openmrcp_asr_resume(switch_asr_handle_t *ah) { - openmrcp_session_t *openmrcp_session = (openmrcp_session_t *) ah->private_info; + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "openmrcp_asr_resume called\n"); return SWITCH_STATUS_SUCCESS; @@ -656,7 +632,6 @@ /*! function to unload a grammar to the asr interface */ static switch_status_t openmrcp_asr_unload_grammar(switch_asr_handle_t *ah, char *grammar) { - openmrcp_session_t *openmrcp_session = (openmrcp_session_t *) ah->private_info; return SWITCH_STATUS_SUCCESS; } @@ -671,7 +646,6 @@ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "openmrcp_asr_close()\n"); - mrcp_client_t *mrcp_client = openmrcp_module.mrcp_client; mrcp_client_context_t *context = openmrcp_module.mrcp_client_context; // TODO!! should we do a switch_pool_clear(switch_memory_pool_t *p) on the pool held @@ -679,7 +653,7 @@ // destroy channel switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Going to DESTROY CHANNEL\n"); - mrcp_client_context_channel_destroy(context, openmrcp_session->client_session, 1); + mrcp_client_context_channel_destroy(context, openmrcp_session->client_session, openmrcp_session->channel); if (openmrcp_session_wait_for_event(openmrcp_session,OPENMRCP_EVENT_CHANNEL_DESTROY,10000) == MRCP_STATUS_SUCCESS) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "OPENMRCP_EVENT_CHANNEL_DESTROY received\n"); } @@ -824,7 +798,6 @@ globals.client_ip = val; } else if (!strcasecmp(var, "server_ip")) { globals.server_ip = val; - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "globals.server_ip: %s\n", globals.server_ip); } else if (!strcasecmp(var, "client_port")) { globals.client_port = (uint32_t) atoi(val); } else if (!strcasecmp(var, "server_port")) {

1 0

[Freeswitch-branches] [commit] r5585 - in freeswitch/branches/greenlizard/src/mod: asr_tts/mod_mrcprec asr_tts/mod_openmrcp xml_int/mod_xml_curl
by Freeswitch SVN 10 Aug '07

10 Aug '07

Author: greenlizard Date: Fri Aug 10 15:30:18 2007 New Revision: 5585 Added: freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/ freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/Makefile freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/mod_openmrcp.c Removed: freeswitch/branches/greenlizard/src/mod/asr_tts/mod_mrcprec/ Modified: freeswitch/branches/greenlizard/src/mod/xml_int/mod_xml_curl/mod_xml_curl.c Log: rename mod_mrcprec to mod_openmrcp Added: freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/Makefile ============================================================================== --- (empty file) +++ freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/Makefile Fri Aug 10 15:30:18 2007 @@ -0,0 +1,33 @@ +# define these targets in your makefile if you wish +# local_all local_depend local_clean depend_install local_install local_distclean local_extraclean: + +# and define these variables to impact your build + +# TO GET THIS TO COMPILE +# hack ../../../../build/modmake.rules and remove the +# -Werror from the ALL_CFLAGS variable. + +# TODO +# - fix all the code that is causing warnings (We are currently working with Joe Stenbrenner from Division Management to have this task outsourced to Bangalore) +# - re-enable the -Werror flag (see above) +# - needs to compile/link against the _freeswitch_ sofia libs (see FIXME) +# - is there an openmrcp dir in the fs tree yet? (see FIXME) + +# FIXME!!! +OPENMRCP_DIR=/usr/src/openmrcp_stable + +OPENMRCP_INCLUDE=-I$(OPENMRCP_DIR)/mrcpcore/client/include/ -I$(OPENMRCP_DIR)/platform/openmrcpclient/include/ -I$(OPENMRCP_DIR)/mediaframe/include/ -I$(OPENMRCP_DIR)/mrcpcore/engine/include/ -I$(OPENMRCP_DIR)/mrcpcore/include/ -I$(OPENMRCP_DIR)/mrcpcore/parser/include/ -I$(OPENMRCP_DIR)/mrcpcore/server/include/ -I$(OPENMRCP_DIR)/mrcpcore/media/include/ -I$(OPENMRCP_DIR)/mrcpcore/util/include -I$(OPENMRCP_DIR)/mrcpcore/resource/include/ + +OPENMRCP_LIBS=$(OPENMRCP_DIR)/platform/openmrcpclient/.libs/libopenmrcpclient.a $(OPENMRCP_DIR)/platform/openmrcpserver/.libs/libopenmrcpserver.a $(OPENMRCP_DIR)/mrcpcore/client/.libs/libmrcpclient.a $(OPENMRCP_DIR)/mediaframe/.libs/libmediaframe.a $(OPENMRCP_DIR)/mrcpcore/engine/.libs/libmrcpengine.a $(OPENMRCP_DIR)/mrcpcore/parser/.libs/libmrcpparser.a $(OPENMRCP_DIR)/mrcpcore/server/.libs/libmrcpserver.a $(OPENMRCP_DIR)/mrcpcore/media/.libs/libmrcpmedia.a $(OPENMRCP_DIR)/mrcpcore/util/.libs/libmrcputil.a $(OPENMRCP_DIR)/mrcpcore/util/.libs/libmrcputil.a $(OPENMRCP_DIR)/mrcpcore/resource/.libs/libmrcpresource.a $(OPENMRCP_DIR)/module/plugin/demo/.libs/libdemoplugin.a $(OPENMRCP_DIR)/module/mrcpv2agent/.libs/libmrcpv2agent.a $(OPENMRCP_DIR)/module/mrcplivemedia/.libs/libmrcplivemedia.a $(OPENMRCP_DIR)/module/mrcpsofiasip/.libs/libmrcpsofiasip.a $(OPENMRCP_DIR)/mrcpcore/.libs/libmrcpcore.a + +MRCP_APR_INCLUDES = -I../../../../libs/apr/include +MRCP_APU_INCLUDES = -I../../../../libs/apr-util/include +MRCP_SOFIA_INCLUDES = -I/usr/local/include/sofia-sip-1.12 # FIXME!! + +LOCAL_CFLAGS= -Wno-unused +LOCAL_CFLAGS+= -Wno-comment -Wno-sign-compare $(OPENMRCP_INCLUDE) $(MRCP_APR_INCLUDES) $(MRCP_APU_INCLUDES) $(MRCP_SOFIA_INCLUDES) + +LOCAL_LDFLAGS=$(OPENMRCP_LIBS) +VERBOSE=TRUE +include ../../../../build/modmake.rules + Added: freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/mod_openmrcp.c ============================================================================== --- (empty file) +++ freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/mod_openmrcp.c Fri Aug 10 15:30:18 2007 @@ -0,0 +1,910 @@ +/* + * FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application + * Copyright (C) 2005/2006, Anthony Minessale II <anthmct(a)yahoo.com> + * + * Version: MPL 1.1 + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application + * + * The Initial Developer of the Original Code is + * Anthony Minessale II <anthmct(a)yahoo.com> + * Portions created by the Initial Developer are Copyright (C) + * the Initial Developer. All Rights Reserved. + * + * Contributor(s): + * + * Traun Leyden <tleyden(a)branchcut.com> + * + * Module which acts as an MRCP client to an MRCP speech recognition + * server. In other words it bridges freeswitch to an external speech + * recognition system. + * + * Uses OpenMrcp (http://wiki.freeswitch.org/wiki/OpenMRCP) as the + * the client library. + * + */ +#ifdef __ICC +#pragma warning (disable:188) +#endif + + +#include "openmrcp_client.h" +#include "mrcp_client_context.h" +#include "mrcp_recognizer.h" +#include "mrcp_generic_header.h" +#include "rtp_session.h" +#include "mrcp_client.h" +#include "mrcp_client_context.h" +#include "mrcp_client_defs.h" +#include "mrcp_client_session.h" +#include "mrcp_client_resource.h" +#include "mrcp_client_signaling_agent.h" +#include "mrcp_media_agent.h" +#include "mrcp_resource.h" +#include "mrcp_consumer_task.h" +#include <apr_general.h> +#include <apr_file_io.h> +#include <apr_thread_proc.h> +#include <apr_thread_cond.h> +#include <apr_strings.h> + +#include <switch.h> + +#define OPENMRCP_WAIT_TIMEOUT 5000 + +SWITCH_MODULE_LOAD_FUNCTION(mod_openmrcp_load); +SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_openmrcp_shutdown); +SWITCH_MODULE_DEFINITION(mod_openmrcp, mod_openmrcp_load, + mod_openmrcp_shutdown, NULL); + +static struct { + char *client_ip; + char *server_ip; + uint32_t client_port; + uint32_t server_port; + uint32_t rtp_port_min; + uint32_t rtp_port_max; +} globals; + + +typedef enum { + OPENMRCP_EVENT_NONE, + OPENMRCP_EVENT_SESSION_INITIATE, + OPENMRCP_EVENT_SESSION_TERMINATE, + OPENMRCP_EVENT_CHANNEL_CREATE, + OPENMRCP_EVENT_CHANNEL_DESTROY, + OPENMRCP_EVENT_CHANNEL_MODIFY +} openmrcp_event_t; + +static const char *openmrcp_event_names[] = { + "NONE", + "SESSION_INITIATE", + "SESSION_TERMINATE", + "CHANNEL_CREATE", + "CHANNEL_DESTROY", + "CHANNEL_MODIFY", +}; + +// TODO: rename this to openmrcp_session_t +typedef struct openmrcp_session_t openmrcp_session_t; +struct openmrcp_session_t { + mrcp_session_t *client_session; + mrcp_client_channel_t *channel; + switch_queue_t *audio_input_queue; + switch_queue_t *event_queue; + mrcp_message_t *mrcp_message_last_rcvd; + audio_source_t *source; + apr_pool_t *pool; + apr_thread_cond_t *wait_object; + apr_thread_mutex_t *wait_object_mutex; + openmrcp_event_t wait_event; + uint32_t flags; + switch_mutex_t *flag_mutex; +}; + +static apr_status_t openmrcp_recognizer_read_frame(audio_source_t *source, media_frame_t *frame); + +static const audio_source_method_set_t audio_source_method_set = { + NULL, + NULL, + NULL, + openmrcp_recognizer_read_frame +}; + +typedef enum { + FLAG_HAS_TEXT = (1 << 0), + FLAG_BARGE = (1 << 1), + FLAG_READY = (1 << 2) +} mrcp_flag_t; + +typedef struct { + mrcp_client_t *mrcp_client; + mrcp_client_context_t *mrcp_client_context; +} openmrcp_module_t; + +static openmrcp_module_t openmrcp_module; + +static void log_callback(const char* message, void* userdata) +{ + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "%s\n", message); +} + +static const char *state_names[] = { + "NOT_READY", + "READY", + "BARGE_IN", + "END_SPEECH", + "STOPPED", + "BARGE_IN_TIMEOUT", + "END_SPEECH_TIMEOUT", + "BEEP" +}; + +static openmrcp_session_t* openmrcp_session_create() +{ + openmrcp_session_t *openmrcp_session; + apr_pool_t *session_pool; + + if(apr_pool_create(&session_pool,NULL) != APR_SUCCESS) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "failed to create session_pool\n"); + return NULL; + } + + openmrcp_session = apr_palloc(session_pool,sizeof(openmrcp_session_t)); + openmrcp_session->pool = session_pool; + openmrcp_session->client_session = NULL; + openmrcp_session->channel = NULL; + openmrcp_session->audio_input_queue = NULL; + openmrcp_session->event_queue = NULL; + openmrcp_session->wait_event = OPENMRCP_EVENT_NONE; + + apr_thread_mutex_create(&openmrcp_session->wait_object_mutex, APR_THREAD_MUTEX_UNNESTED, openmrcp_session->pool); + + apr_thread_cond_create(&openmrcp_session->wait_object, openmrcp_session->pool); + + + /* create an event queue */ + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "creating event queue\n"); + if (switch_queue_create(&openmrcp_session->event_queue, 1000, openmrcp_session->pool)) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "event queue creation failed\n"); + } + + return openmrcp_session; +} + +static mrcp_status_t openmrcp_session_destroy(openmrcp_session_t *openmrcp_session) +{ + if(!openmrcp_session) { + return MRCP_STATUS_FAILURE; + } + + if(openmrcp_session->pool) { + apr_thread_cond_destroy(openmrcp_session->wait_object); + apr_thread_mutex_destroy(openmrcp_session->wait_object_mutex); + apr_pool_destroy(openmrcp_session->pool); + openmrcp_session->pool = NULL; + } + return MRCP_STATUS_SUCCESS; +} + + +static mrcp_status_t openmrcp_session_wait_for_event(openmrcp_session_t *openmrcp_session, openmrcp_event_t openmrcp_event, size_t timeout) +{ + mrcp_status_t status = MRCP_STATUS_SUCCESS; + openmrcp_event_t *popped_event = NULL; + size_t sleep_ms = 100; + + + if(openmrcp_session->event_queue) { + if (switch_queue_trypop(openmrcp_session->event_queue, &popped_event)) { + // most likely this failed because queue is empty. sleep for timeout seconds and try again? + if (timeout > 0) { + if (timeout < sleep_ms) { + switch_sleep(timeout * 1000); // ms->us + timeout = 0; + } + else { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "sleeping %d millieconds\n", sleep_ms); + switch_sleep(sleep_ms * 1000); // ms->us + timeout -= sleep_ms; + } + + // call recursively + // TODO: This is going to end up in a lot of recursion and + // maybe blow the stack .. rethink this approach + return openmrcp_session_wait_for_event(openmrcp_session, openmrcp_event, timeout); + + } + else { + // nothing in queue, no timeout left, return failure + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "nothing in queue, no timeout left, return failure\n"); + return MRCP_STATUS_FAILURE; + } + + } + else { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "popped event\n"); + // check if popped event matches the event we are looking for + if (*popped_event == openmrcp_event) { + // just what we were waiting for! + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "just what we were waiting for! rturn success\n"); + return MRCP_STATUS_SUCCESS; + } + else { + // nothing popped, but maybe there's other things in queue, or something will arrive + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "popped unexpected\n"); + if (!popped_event) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "popped NULL!!\n"); + } + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "popped: %d\n", *popped_event); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "popped name: %s\n", openmrcp_event_names[*popped_event]); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "popped [%s] but was expecting [%s], but maybe there's other things in queue, or something will arrive\n", openmrcp_event_names[*popped_event], openmrcp_event_names[openmrcp_event]); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "calling recursively\n"); + return openmrcp_session_wait_for_event(openmrcp_session, openmrcp_event, timeout); + } + + } + } + else { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "event queue is null\n"); + return MRCP_STATUS_FAILURE; + } + +} + + +static mrcp_status_t openmrcp_session_signal_event(openmrcp_session_t *openmrcp_session, openmrcp_event_t openmrcp_event) +{ + mrcp_status_t status = MRCP_STATUS_SUCCESS; + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Got event: %s\n", openmrcp_event_names[openmrcp_event]); + + + // allocate memory for event + openmrcp_event_t *event2queue = (openmrcp_event_t *) switch_core_alloc(openmrcp_session->pool, sizeof(openmrcp_event_t)); + *event2queue = openmrcp_event; + + // add it to queue + if (switch_queue_trypush(openmrcp_session->event_queue, (void *) event2queue)) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "could not push event to queue\n"); + status = SWITCH_STATUS_GENERR; + } + else { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "pushed event to queue: %s\n", openmrcp_event_names[*event2queue]); + } + + return status; +} + + +static mrcp_status_t openmrcp_on_session_initiate(mrcp_client_context_t *context, mrcp_session_t *session) +{ + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "openmrcp_on_session_initiate called\n"); + openmrcp_session_t *openmrcp_session = mrcp_client_context_session_object_get(session); + if(!openmrcp_session) { + return MRCP_STATUS_FAILURE; + } + return openmrcp_session_signal_event(openmrcp_session,OPENMRCP_EVENT_SESSION_INITIATE); +} + +static mrcp_status_t openmrcp_on_session_terminate(mrcp_client_context_t *context, mrcp_session_t *session) +{ + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "openmrcp_on_session_terminate called\n"); + openmrcp_session_t *openmrcp_session = mrcp_client_context_session_object_get(session); + if(!openmrcp_session) { + return MRCP_STATUS_FAILURE; + } + return openmrcp_session_signal_event(openmrcp_session,OPENMRCP_EVENT_SESSION_TERMINATE); +} + +static mrcp_status_t openmrcp_on_channel_add(mrcp_client_context_t *context, mrcp_session_t *session, mrcp_client_channel_t *channel) +{ + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "openmrcp_on_channel_add called\n"); + openmrcp_session_t *openmrcp_session = mrcp_client_context_session_object_get(session); + if(!openmrcp_session) { + return MRCP_STATUS_FAILURE; + } + return openmrcp_session_signal_event(openmrcp_session,OPENMRCP_EVENT_CHANNEL_CREATE); +} + +static mrcp_status_t openmrcp_on_channel_remove(mrcp_client_context_t *context, mrcp_session_t *session, mrcp_client_channel_t *channel) +{ + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "openmrcp_on_channel_remove called\n"); + openmrcp_session_t *openmrcp_session = mrcp_client_context_session_object_get(session); + if(!openmrcp_session) { + return MRCP_STATUS_FAILURE; + } + return openmrcp_session_signal_event(openmrcp_session,OPENMRCP_EVENT_CHANNEL_DESTROY); +} + +/** this is called by the mrcp core whenever an mrcp message is received from + the other side. */ +static mrcp_status_t openmrcp_on_channel_modify(mrcp_client_context_t *context, mrcp_session_t *session, mrcp_message_t *mrcp_message) +{ + + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "openmrcp_on_channel_modify called\n"); + openmrcp_session_t *openmrcp_session = mrcp_client_context_session_object_get(session); + if(!openmrcp_session) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "!openmrcp_session\n"); + return MRCP_STATUS_FAILURE; + } + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "openmrcp_session: %p\n", openmrcp_session); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "mrcp msg method name: %s\n", mrcp_message->start_line.method_name); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "mrcp msg body: %s\n", mrcp_message->body); + + if (!strcmp(mrcp_message->start_line.method_name,"RECOGNITION-COMPLETE")) { + openmrcp_session->mrcp_message_last_rcvd = mrcp_message; + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "setting FLAG_HAS_TEXT\n"); + switch_set_flag_locked(openmrcp_session, FLAG_HAS_TEXT); + } + else if (!strcmp(mrcp_message->start_line.method_name,"START-OF-SPEECH")) { + openmrcp_session->mrcp_message_last_rcvd = mrcp_message; + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "setting FLAG_BARGE\n"); + switch_set_flag_locked(openmrcp_session, FLAG_BARGE); + } + else { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "ignoring method: %s\n", mrcp_message->start_line.method_name); + } + + return openmrcp_session_signal_event(openmrcp_session,OPENMRCP_EVENT_CHANNEL_MODIFY); +} + + + +static apr_status_t set_default_options(openmrcp_client_options_t *options) +{ + mrcp_logger.priority = MRCP_PRIO_INFO; + options->client_ip = globals.client_ip; + options->server_ip = globals.server_ip; + options->client_port = globals.client_port; + options->server_port = globals.server_port; + options->rtp_port_min = globals.rtp_port_min; + options->rtp_port_max = globals.rtp_port_max; + return APR_SUCCESS; +} + + +/** + * Called back by openmcp client thread every time its ready for more audio + * Reads data that was put into a shared fifo queue upon receiving audio frames + * from asr_feed() + */ +static apr_status_t openmrcp_recognizer_read_frame(audio_source_t *source, media_frame_t *frame) +{ + openmrcp_session_t *openmrcp_session = source->object; + frame->type = MEDIA_FRAME_TYPE_NONE; + apr_status_t result; + media_frame_t *queue_frame = NULL; + + + /* pop next media frame data from incoming queue into frame */ + if(openmrcp_session->audio_input_queue) { + if (switch_queue_trypop(openmrcp_session->audio_input_queue, &queue_frame)) { + // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "could not pop from queue\n"); + result = MRCP_STATUS_FAILURE; + } + else { + frame->codec_frame.size = queue_frame->codec_frame.size; + frame->codec_frame.buffer = queue_frame->codec_frame.buffer; + frame->type = MEDIA_FRAME_TYPE_AUDIO; + result = APR_SUCCESS; + } + } + else { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "no audio_in queue\n"); + result = MRCP_STATUS_FAILURE; + } + + return result; +} + +/** Read in the grammar and construct an MRCP Recognize message that has + The grammar attached as the payload */ +static mrcp_status_t openmrcp_recog_start(mrcp_client_context_t *context, openmrcp_session_t *openmrcp_session, char *path) +{ + + mrcp_generic_header_t *generic_header; + apr_status_t rv; + apr_file_t *fp; + apr_pool_t *mp; + apr_finfo_t finfo; + char *buf1; + int bytes2read = 0; + + mrcp_message_t *mrcp_message = mrcp_client_context_message_get(context, openmrcp_session->client_session, openmrcp_session->channel, RECOGNIZER_RECOGNIZE); + + if(!mrcp_message) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not create mrcp msg\n"); + return MRCP_STATUS_FAILURE; + } + + /* open the file with the grammar and read into char* buffer */ + mp = mrcp_message->pool; + if ((rv = apr_file_open(&fp, path, APR_READ, APR_OS_DEFAULT, mp)) != APR_SUCCESS) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not read grammar\n"); + return -1; + } + rv = apr_file_info_get(&finfo, APR_FINFO_NORM, fp); + + buf1 = apr_palloc(mp, finfo.size); + bytes2read = finfo.size; + rv = apr_file_read(fp, buf1, &bytes2read); + generic_header = mrcp_generic_header_prepare(mrcp_message); + if(!generic_header) { + return MRCP_STATUS_FAILURE; + } + + generic_header->content_type = mrcp_str_pdup(mrcp_message->pool,"application/srgs+xml"); + mrcp_generic_header_property_add(mrcp_message,GENERIC_HEADER_CONTENT_TYPE); + mrcp_message->body = mrcp_str_pdup(mrcp_message->pool,buf1); + + /* send the MRCP RECOGNIZE message to MRCP server */ + return mrcp_client_context_channel_modify(context, openmrcp_session->client_session, mrcp_message); + +} + + +/** + * Freeswitch calls this from switch_ivr_detect_speech() and then adds a media + * bug to tap into the channel's audio, which will result in all data getting + * passed to asr_feed() and calls to asr_check_results() on each recevied frame. + * + * This code expects certain one-time initialization of the openmrcp client + * engine/systeme to have already taken place.function to open the asr interface + */ +static switch_status_t openmrcp_asr_open(switch_asr_handle_t *ah, char *codec, int rate, char *dest, switch_asr_flag_t *flags) +{ + mrcp_client_t *mrcp_client = openmrcp_module.mrcp_client; + mrcp_client_context_t *mrcp_client_context = openmrcp_module.mrcp_client_context ; + openmrcp_client_options_t *options; + openmrcp_session_t *openmrcp_session; + audio_source_t *source; + + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "asr_open called, codec: %s, rate: %d\n", codec, rate); + + /*! + NOTE: According to the current FS media bugs design, the media bug can only feed audio + data in SLIN (L16) format. So we dont need to worry about other codecs. + + NOTE: forcing MRCP to use 20 as the CODEC_FRAME_TIME_BASE effectively ensures + that it matches with 16-bit audio at 8kz with 320 byte frames. in testing, leaving + CODEC_FRAME_TIME_BASE at 10 and using pop (instead of trypop) in + openmrcp_recognizer_read_frame() actually produces clean audio, however it causes + other problems as the full channel/session cleanup never completes in openmrcp, most + likely due to a thread being blocked on a pop call from audio queue. but with trypop + (to avoid the cleanup problem), it only produces clean audio when CODEC_FRAME_TIME_BASE + is set to 20. + */ + if (strcmp(codec,"L16")) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Sorry, only L16 codec supported\n"); + return SWITCH_STATUS_GENERR; + } + if (rate != 8000) { + // TODO: look into supporting other sample rates + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Sorry, only 8kz supported\n"); + return SWITCH_STATUS_GENERR; + } + if (CODEC_FRAME_TIME_BASE != 20) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "You must recompile openmrcp with #define CODEC_FRAME_TIME_BASE 20\n"); + return SWITCH_STATUS_GENERR; + } + else { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "CODEC_FRAME_TIME_BASE: %d\n", CODEC_FRAME_TIME_BASE); + } + + /* create session */ + openmrcp_session = openmrcp_session_create(); + if (!openmrcp_session) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "openmrcp_session creation FAILED\n"); + return SWITCH_STATUS_GENERR; + } + openmrcp_session->client_session = mrcp_client_context_session_create(mrcp_client_context,openmrcp_session); + if (!openmrcp_session->client_session) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "openmrcp_session creation FAILED\n"); + return SWITCH_STATUS_GENERR; + } + + /* create audio source */ + source = mrcp_palloc(openmrcp_session->pool,sizeof(audio_source_t)); + source->method_set = &audio_source_method_set; + source->object = openmrcp_session; + openmrcp_session->source = source; + + /** + * create a new fifo queue. incoming audio received from freeswitch + * will be put into this queue, and it will be later pulled out by + * the openmrcp client thread. + */ + if (switch_queue_create(&openmrcp_session->audio_input_queue, 10000, openmrcp_session->pool)) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "audio queue creation failed\n"); + return SWITCH_STATUS_MEMERR; + } + + openmrcp_session->flags = *flags; + switch_mutex_init(&openmrcp_session->flag_mutex, SWITCH_MUTEX_NESTED, openmrcp_session->pool); + + ah->private_info = openmrcp_session; + + return SWITCH_STATUS_SUCCESS; + +} + +/* function to load a grammar to the asr interface */ +static switch_status_t openmrcp_asr_load_grammar(switch_asr_handle_t *ah, char *grammar, char *path) +{ + /** Read grammar from path and create and send and MRCP RECOGNIZE msg + that has the grammar attached to body. + + TODO: - how does DEFINE-GRAMMAR fit into the picture here? (if at all) + */ + + openmrcp_session_t *openmrcp_session = (openmrcp_session_t *) ah->private_info; + mrcp_client_t *mrcp_client = openmrcp_module.mrcp_client; + mrcp_client_context_t *mrcp_client_context = openmrcp_module.mrcp_client_context; + audio_source_t *source = openmrcp_session->source; + + /* create recognizer channel, also starts outgoing rtp media */ + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "Loading grammar\n"); + + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Create Recognizer Channel\n"); + openmrcp_session->channel = mrcp_client_recognizer_channel_create(mrcp_client_context, openmrcp_session->client_session, source); + + if (!openmrcp_session->channel) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Failed to create recognizer channel\n"); + return SWITCH_STATUS_FALSE; + } + + /* wait for recognizer channel creation */ + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "WAITING FOR CHAN CREATE\n"); + if(openmrcp_session_wait_for_event(openmrcp_session, OPENMRCP_EVENT_CHANNEL_CREATE, OPENMRCP_WAIT_TIMEOUT) == MRCP_STATUS_SUCCESS) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Got channel creation event\n"); + + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "WAITING FOR NONE\n"); + if (openmrcp_session_wait_for_event(openmrcp_session,OPENMRCP_EVENT_NONE,200) == MRCP_STATUS_SUCCESS) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "GOT NONE EVENT\n"); + } + else { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "NEVER GOT NONE EVENT\n"); + } + + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Start Recognizer\n"); + openmrcp_recog_start(mrcp_client_context, openmrcp_session, path); + + } + else { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "never got channel creation event\n"); + } + + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "Finished loading grammar\n"); + return SWITCH_STATUS_SUCCESS; +} + + +/*! function to feed audio to the ASR*/ +static switch_status_t openmrcp_asr_feed(switch_asr_handle_t *ah, void *data, unsigned int len, switch_asr_flag_t *flags) +{ + + openmrcp_session_t *openmrcp_session = (openmrcp_session_t *) ah->private_info; + mrcp_client_t *mrcp_client = openmrcp_module.mrcp_client; + mrcp_client_context_t *mrcp_client_context = openmrcp_module.mrcp_client_context; + media_frame_t *media_frame; + switch_byte_t *buffer; + + /* create new media frame */ + media_frame = (media_frame_t *) switch_core_alloc(openmrcp_session->pool, sizeof(media_frame_t)); + if (!media_frame) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "media_frame creation failed\n"); + return SWITCH_STATUS_MEMERR; + } + + /** + * since *data buffer might get freed by caller (true or false?), allocate a + * new buffer and copy *data into it. + * + * MAJOR DESIGN ISSUE!! It is way too expensive to be calling malloc on every audio frame, + * this needs to send the packet directly. OpenMrcp will need a way to disable their own + * timer so that the fs timer is the only one driving the media. + **/ + buffer = (switch_byte_t *) switch_core_alloc(openmrcp_session->pool, sizeof(switch_byte_t)*len); + if (!buffer) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not allocate buffer\n"); + return SWITCH_STATUS_MEMERR; + } + buffer = memcpy(buffer, data, len); + + media_frame->codec_frame.buffer = buffer; + media_frame->codec_frame.size = len; + media_frame->type = MEDIA_FRAME_TYPE_AUDIO; + + /* push audio to queue */ + if (switch_queue_trypush(openmrcp_session->audio_input_queue, (void *) media_frame)) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "could not push audio to queue\n"); + return SWITCH_STATUS_GENERR; + } + + return SWITCH_STATUS_SUCCESS; +} + + +static switch_status_t openmrcp_asr_pause(switch_asr_handle_t *ah) +{ + openmrcp_session_t *openmrcp_session = (openmrcp_session_t *) ah->private_info; + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "openmrcp_asr_pause called\n"); + + return SWITCH_STATUS_SUCCESS; + +} + +static switch_status_t openmrcp_asr_resume(switch_asr_handle_t *ah) +{ + openmrcp_session_t *openmrcp_session = (openmrcp_session_t *) ah->private_info; + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "openmrcp_asr_resume called\n"); + + return SWITCH_STATUS_SUCCESS; +} + + +/*! function to unload a grammar to the asr interface */ +static switch_status_t openmrcp_asr_unload_grammar(switch_asr_handle_t *ah, char *grammar) +{ + openmrcp_session_t *openmrcp_session = (openmrcp_session_t *) ah->private_info; + + return SWITCH_STATUS_SUCCESS; +} + +/** + * Freeswitch calls this whenever the channel is hungup or the + * speech detection is stopped via a call to switch_ivr_stop_detect_speech() + */ +static switch_status_t openmrcp_asr_close(switch_asr_handle_t *ah, switch_asr_flag_t *flags) +{ + openmrcp_session_t *openmrcp_session = (openmrcp_session_t *) ah->private_info; + + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "openmrcp_asr_close()\n"); + + mrcp_client_t *mrcp_client = openmrcp_module.mrcp_client; + mrcp_client_context_t *context = openmrcp_module.mrcp_client_context; + + // TODO!! should we do a switch_pool_clear(switch_memory_pool_t *p) on the pool held + // by openmrcp_session? + + // destroy channel + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Going to DESTROY CHANNEL\n"); + mrcp_client_context_channel_destroy(context, openmrcp_session->client_session, 1); + if (openmrcp_session_wait_for_event(openmrcp_session,OPENMRCP_EVENT_CHANNEL_DESTROY,10000) == MRCP_STATUS_SUCCESS) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "OPENMRCP_EVENT_CHANNEL_DESTROY received\n"); + } + else { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "timed out waiting for OPENMRCP_EVENT_CHANNEL_DESTROY\n"); + } + + // terminate client session + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Going to TERMINATE SESSION\n"); + mrcp_client_context_session_terminate(context, openmrcp_session->client_session); + if (openmrcp_session_wait_for_event(openmrcp_session,OPENMRCP_EVENT_SESSION_TERMINATE,10000) == MRCP_STATUS_SUCCESS) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "OPENMRCP_EVENT_SESSION_TERMINATE recevied\n"); + } + else { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "timed out waiting for OPENMRCP_EVENT_SESSION_TERMINATE\n"); + } + + // destroy client session (NOTE: this sends a BYE to the other side) + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Going to DESTROY SESSION\n"); + mrcp_client_context_session_destroy(context, openmrcp_session->client_session); + + // destroys the openmrcp_session struct + openmrcp_session_destroy(openmrcp_session); + + switch_set_flag(ah, SWITCH_ASR_FLAG_CLOSED); + + return SWITCH_STATUS_SUCCESS; +} + + +/** + * Freeswitch calls this method from the speech_thread() thread body method + * in switch_ivr_async.c every time a new frame is received by the media bug + * attached to the audio channel. If this method returns SWITCH_STATUS_SUCCESS, + * then Freeswitch will call openmrcp_asr_get_results() to get the result value. + */ +static switch_status_t openmrcp_asr_check_results(switch_asr_handle_t *ah, switch_asr_flag_t *flags) +{ + openmrcp_session_t *openmrcp_session = (openmrcp_session_t *) ah->private_info; + + switch_status_t rv = (switch_test_flag(openmrcp_session, FLAG_HAS_TEXT) || switch_test_flag(openmrcp_session, FLAG_BARGE)) ? SWITCH_STATUS_SUCCESS : SWITCH_STATUS_FALSE; + + return rv; +} + +/*! This will be called after asr_check_results returns SUCCESS */ +static switch_status_t openmrcp_asr_get_results(switch_asr_handle_t *ah, char **xmlstr, switch_asr_flag_t *flags) +{ + openmrcp_session_t *openmrcp_session = (openmrcp_session_t *) ah->private_info; + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "openmrcp_asr_get_results called\n"); + switch_status_t ret = SWITCH_STATUS_SUCCESS; + + if (switch_test_flag(openmrcp_session, FLAG_BARGE)) { + switch_clear_flag_locked(openmrcp_session, FLAG_BARGE); + ret = SWITCH_STATUS_BREAK; + } + + if (switch_test_flag(openmrcp_session, FLAG_HAS_TEXT)) { + /*! + we have to extract the XML but stripping off the <?xml version="1.0"?> + header. the body looks like: + + Completion-Cause:001 no-match + Content-Type: application/nlsml+xml + Content-Length: 260 + + <?xml version="1.0"?> + <result xmlns="http://www.ietf.org/xml/ns/mrcpv2" xmlns:ex="http://www.example.com/example" score="100" grammar="session:request1@form-level.store"> + <interpretation> + <input mode="speech">open a</input> + </interpretation> + </result> + */ + + char *marker = "?>"; // FIXME -- lame and brittle way of doing this. use regex or better. + char *position = strstr(openmrcp_session->mrcp_message_last_rcvd->body, marker); + if (!position) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Bad result received from mrcp server: %s", openmrcp_session->mrcp_message_last_rcvd->body); + ret = SWITCH_STATUS_FALSE; + } + else { + position += strlen(marker); + *xmlstr = strdup(position); + } + + // since we are returning our result here, future calls to check_results + // should return False + switch_clear_flag_locked(openmrcp_session, FLAG_HAS_TEXT); + ret = SWITCH_STATUS_SUCCESS; + } + return ret; +} + +static switch_asr_interface_t openmrcp_asr_interface = { + /*.interface_name*/ "openmrcp", + /*.asr_open*/ openmrcp_asr_open, + /*.asr_load_grammar*/ openmrcp_asr_load_grammar, + /*.asr_unload_grammar*/ openmrcp_asr_unload_grammar, + /*.asr_close*/ openmrcp_asr_close, + /*.asr_feed*/ openmrcp_asr_feed, + /*.asr_resume*/ openmrcp_asr_resume, + /*.asr_pause*/ openmrcp_asr_pause, + /*.asr_check_results*/ openmrcp_asr_check_results, + /*.asr_get_results*/ openmrcp_asr_get_results +}; + +static switch_loadable_module_interface_t openmrcp_module_interface = { + /*.module_name */ modname, + /*.endpoint_interface */ NULL, + /*.timer_interface */ NULL, + /*.dialplan_interface */ NULL, + /*.codec_interface */ NULL, + /*.application_interface */ NULL, + /*.api_interface */ NULL, + /*.file_interface */ NULL, + /*.speech_interface */ NULL, + /*.directory_interface */ NULL, + /*.chat_interface */ NULL, + /*.say_interface */ NULL, + /*.asr_interface */ &openmrcp_asr_interface +}; + + +static switch_status_t do_config(void) +{ + char *cf = "mod_openmrcp.conf"; + switch_xml_t cfg, xml, settings, param; + + if (!(xml = switch_xml_open_cfg(cf, &cfg, NULL))) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "open of %s failed\n", cf); + return SWITCH_STATUS_TERM; + } + + memset(&globals,0,sizeof(globals)); + + if ((settings = switch_xml_child(cfg, "settings"))) { + for (param = switch_xml_child(settings, "param"); param; param = param->next) { + char *var = (char *) switch_xml_attr_soft(param, "name"); + char *val = (char *) switch_xml_attr_soft(param, "value"); + + if (!strcasecmp(var, "client_ip")) { + globals.client_ip = val; + } else if (!strcasecmp(var, "server_ip")) { + globals.server_ip = val; + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "globals.server_ip: %s\n", globals.server_ip); + } else if (!strcasecmp(var, "client_port")) { + globals.client_port = (uint32_t) atoi(val); + } else if (!strcasecmp(var, "server_port")) { + globals.server_port = (uint32_t) atoi(val); + } else if (!strcasecmp(var, "rtp_port_min")) { + globals.rtp_port_min = (uint32_t) atoi(val); + } else if (!strcasecmp(var, "rtp_port_max")) { + globals.rtp_port_max = (uint32_t) atoi(val); + } + + } + } + + return SWITCH_STATUS_SUCCESS; +} + + +static switch_status_t mrcp_init() +{ + /*! + Perform one-time initialization of client library + */ + + mrcp_mem_pool_t *pool; + mrcp_client_event_handler_t *client_event_handler; + mrcp_client_t *mrcp_client; + mrcp_client_context_t *mrcp_client_context; + openmrcp_client_options_t *options; + + pool = mrcp_global_pool_get(); + options = mrcp_palloc(pool,sizeof(openmrcp_client_options_t)); + client_event_handler = mrcp_palloc(pool,sizeof(mrcp_client_event_handler_t)); + set_default_options(options); + + client_event_handler->on_session_initiate = openmrcp_on_session_initiate; + client_event_handler->on_session_terminate = openmrcp_on_session_terminate; + client_event_handler->on_channel_add = openmrcp_on_channel_add; + client_event_handler->on_channel_remove = openmrcp_on_channel_remove; + client_event_handler->on_channel_modify = openmrcp_on_channel_modify; + + // create client context, which to must be passed to client engine + mrcp_client_context = mrcp_client_context_create(&openmrcp_module,client_event_handler); + if(!mrcp_client_context) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "mrcp_client_context creation failed\n"); + return SWITCH_STATUS_GENERR; + } + openmrcp_module.mrcp_client_context = mrcp_client_context; + + // this basically starts a thread that pulls events from the event queue + // and handles them + mrcp_client = openmrcp_client_start(options,mrcp_client_context); + if(!mrcp_client) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "openmrcp_client_start FAILED\n"); + mrcp_client_context_destroy(mrcp_client_context); + return SWITCH_STATUS_GENERR; + } + + openmrcp_module.mrcp_client = mrcp_client; + + return SWITCH_STATUS_SUCCESS; +} + +SWITCH_MODULE_LOAD_FUNCTION(mod_openmrcp_load) +{ + + /* connect my internal structure to the blank pointer passed to me */ + *module_interface = &openmrcp_module_interface; + + /* read config */ + do_config(); + + /* initialize openmrcp */ + mrcp_global_init(); + mrcp_init(); + + /* indicate that the module should continue to be loaded */ + return SWITCH_STATUS_SUCCESS; +} + +SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_openmrcp_shutdown) +{ + return SWITCH_STATUS_UNLOAD; +} Modified: freeswitch/branches/greenlizard/src/mod/xml_int/mod_xml_curl/mod_xml_curl.c ============================================================================== --- freeswitch/branches/greenlizard/src/mod/xml_int/mod_xml_curl/mod_xml_curl.c (original) +++ freeswitch/branches/greenlizard/src/mod/xml_int/mod_xml_curl/mod_xml_curl.c Fri Aug 10 15:30:18 2007 @@ -72,6 +72,7 @@ xml_binding_t *binding = (xml_binding_t *) user_data; char *file_url; struct curl_slist *slist = NULL; + long httpRes; if (!binding) { return NULL; @@ -126,6 +127,7 @@ } curl_easy_perform(curl_handle); + curl_easy_getinfo(curl_handle, CURLINFO_RESPONSE_CODE,&httpRes); curl_easy_cleanup(curl_handle); close(config_data.fd); } else { @@ -134,8 +136,14 @@ switch_safe_free(data); - if (!(xml = switch_xml_parse_file(filename))) { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error Parsing Result!\n"); + if(httpRes==200) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Got 200 response, calling switch_xml_parse_file: %s!\n", filename); + if (!(xml = switch_xml_parse_file(filename))) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error Parsing Result!\n"); + } + } else { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Received HTTP error %ld trying to fetch %s\n",httpRes,key_value); + xml=NULL; } unlink(filename);

1 0

[Freeswitch-branches] [commit] r5584 - freeswitch/branches/greenlizard/src/mod/asr_tts/mod_mrcprec
by Freeswitch SVN 10 Aug '07

10 Aug '07

Author: greenlizard Date: Fri Aug 10 14:24:42 2007 New Revision: 5584 Modified: freeswitch/branches/greenlizard/src/mod/asr_tts/mod_mrcprec/Makefile freeswitch/branches/greenlizard/src/mod/asr_tts/mod_mrcprec/mod_mrcprec.c Log: Use configuration file instead of hardcoding parameters Modified: freeswitch/branches/greenlizard/src/mod/asr_tts/mod_mrcprec/Makefile ============================================================================== --- freeswitch/branches/greenlizard/src/mod/asr_tts/mod_mrcprec/Makefile (original) +++ freeswitch/branches/greenlizard/src/mod/asr_tts/mod_mrcprec/Makefile Fri Aug 10 14:24:42 2007 @@ -3,12 +3,18 @@ # and define these variables to impact your build +# TO GET THIS TO COMPILE +# hack ../../../../build/modmake.rules and remove the +# -Werror from the ALL_CFLAGS variable. + # TODO -# - needs to compile/link against the freeswitch sofia libs -# - tie up these includes somehow so there isn't so many -# - ditto for the .a library files +# - fix all the code that is causing warnings (We are currently working with Joe Stenbrenner from Division Management to have this task outsourced to Bangalore) +# - re-enable the -Werror flag (see above) +# - needs to compile/link against the _freeswitch_ sofia libs (see FIXME) +# - is there an openmrcp dir in the fs tree yet? (see FIXME) -OPENMRCP_DIR=/usr/src/openmrcp_stable # FIXME!!! +# FIXME!!! +OPENMRCP_DIR=/usr/src/openmrcp_stable OPENMRCP_INCLUDE=-I$(OPENMRCP_DIR)/mrcpcore/client/include/ -I$(OPENMRCP_DIR)/platform/openmrcpclient/include/ -I$(OPENMRCP_DIR)/mediaframe/include/ -I$(OPENMRCP_DIR)/mrcpcore/engine/include/ -I$(OPENMRCP_DIR)/mrcpcore/include/ -I$(OPENMRCP_DIR)/mrcpcore/parser/include/ -I$(OPENMRCP_DIR)/mrcpcore/server/include/ -I$(OPENMRCP_DIR)/mrcpcore/media/include/ -I$(OPENMRCP_DIR)/mrcpcore/util/include -I$(OPENMRCP_DIR)/mrcpcore/resource/include/ Modified: freeswitch/branches/greenlizard/src/mod/asr_tts/mod_mrcprec/mod_mrcprec.c ============================================================================== --- freeswitch/branches/greenlizard/src/mod/asr_tts/mod_mrcprec/mod_mrcprec.c (original) +++ freeswitch/branches/greenlizard/src/mod/asr_tts/mod_mrcprec/mod_mrcprec.c Fri Aug 10 14:24:42 2007 @@ -67,6 +67,16 @@ SWITCH_MODULE_DEFINITION(mod_mrcprec, mod_mrcprec_load, mod_mrcprec_shutdown, NULL); +static struct { + char *client_ip; + char *server_ip; + uint32_t client_port; + uint32_t server_port; + uint32_t rtp_port_min; + uint32_t rtp_port_max; +} globals; + + typedef enum { OPENMRCP_EVENT_NONE, OPENMRCP_EVENT_SESSION_INITIATE, @@ -194,24 +204,18 @@ openmrcp_event_t *popped_event = NULL; size_t sleep_ms = 100; - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "openmrcp_session_wait_for_event()\n"); if(openmrcp_session->event_queue) { if (switch_queue_trypop(openmrcp_session->event_queue, &popped_event)) { - // most likely this failed because queue is empty. how about we sleep for - // timeout seconds and try again? - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "did not pop anything from event queue\n"); - + // most likely this failed because queue is empty. sleep for timeout seconds and try again? if (timeout > 0) { - if (timeout < sleep_ms) { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "sleeping %d millieconds\n", timeout); - switch_sleep(timeout * 1000); // convert milliseconds -> microseconds + switch_sleep(timeout * 1000); // ms->us timeout = 0; } else { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "sleeping %d millieconds\n", sleep_ms); - switch_sleep(sleep_ms * 1000); // convert milliseconds -> microseconds + switch_sleep(sleep_ms * 1000); // ms->us timeout -= sleep_ms; } @@ -355,39 +359,20 @@ } -/* this needs to be moved to a config file */ + static apr_status_t set_default_options(openmrcp_client_options_t *options) { - options->client_ip = NULL; - options->client_port = 0; - options->server_ip = NULL; - options->server_port = 0; - options->rtp_port_min = 0; - options->rtp_port_max = 0; mrcp_logger.priority = MRCP_PRIO_INFO; - - if(options->client_ip == NULL) { - options->client_ip = "127.0.0.1"; - } - if(options->client_port == 0) { - options->client_port = 8062; - } - if(options->server_ip == NULL) { - options->server_ip = "127.0.0.1"; - } - if(options->server_port == 0) { - options->server_port = 8060; - } - if(options->rtp_port_min == 0) { - options->rtp_port_min = 4000; - } - if(options->rtp_port_max == 0) { - options->rtp_port_max = 6000; - } - + options->client_ip = globals.client_ip; + options->server_ip = globals.server_ip; + options->client_port = globals.client_port; + options->server_port = globals.server_port; + options->rtp_port_min = globals.rtp_port_min; + options->rtp_port_max = globals.rtp_port_max; return APR_SUCCESS; } + /** * Called back by openmcp client thread every time its ready for more audio * Reads data that was put into a shared fifo queue upon receiving audio frames @@ -487,22 +472,24 @@ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "asr_open called, codec: %s, rate: %d\n", codec, rate); /*! - sanity checking regarding codec/rate - NOTE: some of the other codecs and rates might - actually work, but since they have not been tested yet, they are currently not - allowed. forcing MRCP to use 20 as the CODEC_FRAME_TIME_BASE effectively ensures + NOTE: According to the current FS media bugs design, the media bug can only feed audio + data in SLIN (L16) format. So we dont need to worry about other codecs. + + NOTE: forcing MRCP to use 20 as the CODEC_FRAME_TIME_BASE effectively ensures that it matches with 16-bit audio at 8kz with 320 byte frames. in testing, leaving CODEC_FRAME_TIME_BASE at 10 and using pop (instead of trypop) in openmrcp_recognizer_read_frame() actually produces clean audio, however it causes other problems as the full channel/session cleanup never completes in openmrcp, most likely due to a thread being blocked on a pop call from audio queue. but with trypop (to avoid the cleanup problem), it only produces clean audio when CODEC_FRAME_TIME_BASE - is set to 20. + is set to 20. */ if (strcmp(codec,"L16")) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Sorry, only L16 codec supported\n"); return SWITCH_STATUS_GENERR; } if (rate != 8000) { + // TODO: look into supporting other sample rates switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Sorry, only 8kz supported\n"); return SWITCH_STATUS_GENERR; } @@ -619,8 +606,14 @@ return SWITCH_STATUS_MEMERR; } - /** since *data buffer might get freed by caller (true or false?), allocate a - new buffer and copy *data into it **/ + /** + * since *data buffer might get freed by caller (true or false?), allocate a + * new buffer and copy *data into it. + * + * MAJOR DESIGN ISSUE!! It is way too expensive to be calling malloc on every audio frame, + * this needs to send the packet directly. OpenMrcp will need a way to disable their own + * timer so that the fs timer is the only one driving the media. + **/ buffer = (switch_byte_t *) switch_core_alloc(openmrcp_session->pool, sizeof(switch_byte_t)*len); if (!buffer) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not allocate buffer\n"); @@ -810,6 +803,45 @@ }; +static switch_status_t do_config(void) +{ + char *cf = "mod_openmrcp.conf"; + switch_xml_t cfg, xml, settings, param; + + if (!(xml = switch_xml_open_cfg(cf, &cfg, NULL))) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "open of %s failed\n", cf); + return SWITCH_STATUS_TERM; + } + + memset(&globals,0,sizeof(globals)); + + if ((settings = switch_xml_child(cfg, "settings"))) { + for (param = switch_xml_child(settings, "param"); param; param = param->next) { + char *var = (char *) switch_xml_attr_soft(param, "name"); + char *val = (char *) switch_xml_attr_soft(param, "value"); + + if (!strcasecmp(var, "client_ip")) { + globals.client_ip = val; + } else if (!strcasecmp(var, "server_ip")) { + globals.server_ip = val; + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "globals.server_ip: %s\n", globals.server_ip); + } else if (!strcasecmp(var, "client_port")) { + globals.client_port = (uint32_t) atoi(val); + } else if (!strcasecmp(var, "server_port")) { + globals.server_port = (uint32_t) atoi(val); + } else if (!strcasecmp(var, "rtp_port_min")) { + globals.rtp_port_min = (uint32_t) atoi(val); + } else if (!strcasecmp(var, "rtp_port_max")) { + globals.rtp_port_max = (uint32_t) atoi(val); + } + + } + } + + return SWITCH_STATUS_SUCCESS; +} + + static switch_status_t mrcp_init() { /*! @@ -860,6 +892,9 @@ /* connect my internal structure to the blank pointer passed to me */ *module_interface = &mrcprec_module_interface; + + /* read config */ + do_config(); /* initialize openmrcp */ mrcp_global_init();

1 0

[Freeswitch-branches] [commit] r5582 - freeswitch/branches/greenlizard/src/mod/asr_tts/mod_mrcprec
by Freeswitch SVN 09 Aug '07

09 Aug '07

Author: greenlizard Date: Thu Aug 9 17:24:12 2007 New Revision: 5582 Added: freeswitch/branches/greenlizard/src/mod/asr_tts/mod_mrcprec/ freeswitch/branches/greenlizard/src/mod/asr_tts/mod_mrcprec/Makefile freeswitch/branches/greenlizard/src/mod/asr_tts/mod_mrcprec/mod_mrcprec.c Log: rough (but working) version of a module which connects to a speech recognition server via openmrcp client lib Added: freeswitch/branches/greenlizard/src/mod/asr_tts/mod_mrcprec/Makefile ============================================================================== --- (empty file) +++ freeswitch/branches/greenlizard/src/mod/asr_tts/mod_mrcprec/Makefile Thu Aug 9 17:24:12 2007 @@ -0,0 +1,27 @@ +# define these targets in your makefile if you wish +# local_all local_depend local_clean depend_install local_install local_distclean local_extraclean: + +# and define these variables to impact your build + +# TODO +# - needs to compile/link against the freeswitch sofia libs +# - tie up these includes somehow so there isn't so many +# - ditto for the .a library files + +OPENMRCP_DIR=/usr/src/openmrcp_stable # FIXME!!! + +OPENMRCP_INCLUDE=-I$(OPENMRCP_DIR)/mrcpcore/client/include/ -I$(OPENMRCP_DIR)/platform/openmrcpclient/include/ -I$(OPENMRCP_DIR)/mediaframe/include/ -I$(OPENMRCP_DIR)/mrcpcore/engine/include/ -I$(OPENMRCP_DIR)/mrcpcore/include/ -I$(OPENMRCP_DIR)/mrcpcore/parser/include/ -I$(OPENMRCP_DIR)/mrcpcore/server/include/ -I$(OPENMRCP_DIR)/mrcpcore/media/include/ -I$(OPENMRCP_DIR)/mrcpcore/util/include -I$(OPENMRCP_DIR)/mrcpcore/resource/include/ + +OPENMRCP_LIBS=$(OPENMRCP_DIR)/platform/openmrcpclient/.libs/libopenmrcpclient.a $(OPENMRCP_DIR)/platform/openmrcpserver/.libs/libopenmrcpserver.a $(OPENMRCP_DIR)/mrcpcore/client/.libs/libmrcpclient.a $(OPENMRCP_DIR)/mediaframe/.libs/libmediaframe.a $(OPENMRCP_DIR)/mrcpcore/engine/.libs/libmrcpengine.a $(OPENMRCP_DIR)/mrcpcore/parser/.libs/libmrcpparser.a $(OPENMRCP_DIR)/mrcpcore/server/.libs/libmrcpserver.a $(OPENMRCP_DIR)/mrcpcore/media/.libs/libmrcpmedia.a $(OPENMRCP_DIR)/mrcpcore/util/.libs/libmrcputil.a $(OPENMRCP_DIR)/mrcpcore/util/.libs/libmrcputil.a $(OPENMRCP_DIR)/mrcpcore/resource/.libs/libmrcpresource.a $(OPENMRCP_DIR)/module/plugin/demo/.libs/libdemoplugin.a $(OPENMRCP_DIR)/module/mrcpv2agent/.libs/libmrcpv2agent.a $(OPENMRCP_DIR)/module/mrcplivemedia/.libs/libmrcplivemedia.a $(OPENMRCP_DIR)/module/mrcpsofiasip/.libs/libmrcpsofiasip.a $(OPENMRCP_DIR)/mrcpcore/.libs/libmrcpcore.a + +MRCP_APR_INCLUDES = -I../../../../libs/apr/include +MRCP_APU_INCLUDES = -I../../../../libs/apr-util/include +MRCP_SOFIA_INCLUDES = -I/usr/local/include/sofia-sip-1.12 # FIXME!! + +LOCAL_CFLAGS= -Wno-unused +LOCAL_CFLAGS+= -Wno-comment -Wno-sign-compare $(OPENMRCP_INCLUDE) $(MRCP_APR_INCLUDES) $(MRCP_APU_INCLUDES) $(MRCP_SOFIA_INCLUDES) + +LOCAL_LDFLAGS=$(OPENMRCP_LIBS) +VERBOSE=TRUE +include ../../../../build/modmake.rules + Added: freeswitch/branches/greenlizard/src/mod/asr_tts/mod_mrcprec/mod_mrcprec.c ============================================================================== --- (empty file) +++ freeswitch/branches/greenlizard/src/mod/asr_tts/mod_mrcprec/mod_mrcprec.c Thu Aug 9 17:24:12 2007 @@ -0,0 +1,875 @@ +/* + * FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application + * Copyright (C) 2005/2006, Anthony Minessale II <anthmct(a)yahoo.com> + * + * Version: MPL 1.1 + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application + * + * The Initial Developer of the Original Code is + * Anthony Minessale II <anthmct(a)yahoo.com> + * Portions created by the Initial Developer are Copyright (C) + * the Initial Developer. All Rights Reserved. + * + * Contributor(s): + * + * Traun Leyden <tleyden(a)branchcut.com> + * + * Module which acts as an MRCP client to an MRCP speech recognition + * server. In other words it bridges freeswitch to an external speech + * recognition system. + * + * Uses OpenMrcp (http://wiki.freeswitch.org/wiki/OpenMRCP) as the + * the client library. + * + */ +#ifdef __ICC +#pragma warning (disable:188) +#endif + + +#include "openmrcp_client.h" +#include "mrcp_client_context.h" +#include "mrcp_recognizer.h" +#include "mrcp_generic_header.h" +#include "rtp_session.h" +#include "mrcp_client.h" +#include "mrcp_client_context.h" +#include "mrcp_client_defs.h" +#include "mrcp_client_session.h" +#include "mrcp_client_resource.h" +#include "mrcp_client_signaling_agent.h" +#include "mrcp_media_agent.h" +#include "mrcp_resource.h" +#include "mrcp_consumer_task.h" +#include <apr_general.h> +#include <apr_file_io.h> +#include <apr_thread_proc.h> +#include <apr_thread_cond.h> +#include <apr_strings.h> + +#include <switch.h> + +#define OPENMRCP_WAIT_TIMEOUT 5000 + +SWITCH_MODULE_LOAD_FUNCTION(mod_mrcprec_load); +SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_mrcprec_shutdown); +SWITCH_MODULE_DEFINITION(mod_mrcprec, mod_mrcprec_load, + mod_mrcprec_shutdown, NULL); + +typedef enum { + OPENMRCP_EVENT_NONE, + OPENMRCP_EVENT_SESSION_INITIATE, + OPENMRCP_EVENT_SESSION_TERMINATE, + OPENMRCP_EVENT_CHANNEL_CREATE, + OPENMRCP_EVENT_CHANNEL_DESTROY, + OPENMRCP_EVENT_CHANNEL_MODIFY +} openmrcp_event_t; + +static const char *openmrcp_event_names[] = { + "NONE", + "SESSION_INITIATE", + "SESSION_TERMINATE", + "CHANNEL_CREATE", + "CHANNEL_DESTROY", + "CHANNEL_MODIFY", +}; + +// TODO: rename this to mrcprec_session_t +typedef struct openmrcp_session_t openmrcp_session_t; +struct openmrcp_session_t { + mrcp_session_t *client_session; + mrcp_client_channel_t *channel; + switch_queue_t *audio_input_queue; + switch_queue_t *event_queue; + mrcp_message_t *mrcp_message_last_rcvd; + audio_source_t *source; + apr_pool_t *pool; + apr_thread_cond_t *wait_object; + apr_thread_mutex_t *wait_object_mutex; + openmrcp_event_t wait_event; + uint32_t flags; + switch_mutex_t *flag_mutex; +}; + +static apr_status_t openmrcp_recognizer_read_frame(audio_source_t *source, media_frame_t *frame); + +static const audio_source_method_set_t audio_source_method_set = { + NULL, + NULL, + NULL, + openmrcp_recognizer_read_frame +}; + +typedef enum { + FLAG_HAS_TEXT = (1 << 0), + FLAG_BARGE = (1 << 1), + FLAG_READY = (1 << 2) +} mrcp_flag_t; + +typedef struct { + mrcp_client_t *mrcp_client; + mrcp_client_context_t *mrcp_client_context; +} mrcprec_module_t; + +static mrcprec_module_t mrcprec_module; + +static void log_callback(const char* message, void* userdata) +{ + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "%s\n", message); +} + +static const char *state_names[] = { + "NOT_READY", + "READY", + "BARGE_IN", + "END_SPEECH", + "STOPPED", + "BARGE_IN_TIMEOUT", + "END_SPEECH_TIMEOUT", + "BEEP" +}; + +static openmrcp_session_t* openmrcp_session_create() +{ + openmrcp_session_t *openmrcp_session; + apr_pool_t *session_pool; + + if(apr_pool_create(&session_pool,NULL) != APR_SUCCESS) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "failed to create session_pool\n"); + return NULL; + } + + openmrcp_session = apr_palloc(session_pool,sizeof(openmrcp_session_t)); + openmrcp_session->pool = session_pool; + openmrcp_session->client_session = NULL; + openmrcp_session->channel = NULL; + openmrcp_session->audio_input_queue = NULL; + openmrcp_session->event_queue = NULL; + openmrcp_session->wait_event = OPENMRCP_EVENT_NONE; + + apr_thread_mutex_create(&openmrcp_session->wait_object_mutex, APR_THREAD_MUTEX_UNNESTED, openmrcp_session->pool); + + apr_thread_cond_create(&openmrcp_session->wait_object, openmrcp_session->pool); + + + /* create an event queue */ + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "creating event queue\n"); + if (switch_queue_create(&openmrcp_session->event_queue, 1000, openmrcp_session->pool)) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "event queue creation failed\n"); + } + + return openmrcp_session; +} + +static mrcp_status_t openmrcp_session_destroy(openmrcp_session_t *openmrcp_session) +{ + if(!openmrcp_session) { + return MRCP_STATUS_FAILURE; + } + + if(openmrcp_session->pool) { + apr_thread_cond_destroy(openmrcp_session->wait_object); + apr_thread_mutex_destroy(openmrcp_session->wait_object_mutex); + apr_pool_destroy(openmrcp_session->pool); + openmrcp_session->pool = NULL; + } + return MRCP_STATUS_SUCCESS; +} + + +static mrcp_status_t openmrcp_session_wait_for_event(openmrcp_session_t *openmrcp_session, openmrcp_event_t openmrcp_event, size_t timeout) +{ + mrcp_status_t status = MRCP_STATUS_SUCCESS; + openmrcp_event_t *popped_event = NULL; + size_t sleep_ms = 100; + + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "openmrcp_session_wait_for_event()\n"); + + if(openmrcp_session->event_queue) { + if (switch_queue_trypop(openmrcp_session->event_queue, &popped_event)) { + // most likely this failed because queue is empty. how about we sleep for + // timeout seconds and try again? + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "did not pop anything from event queue\n"); + + if (timeout > 0) { + + if (timeout < sleep_ms) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "sleeping %d millieconds\n", timeout); + switch_sleep(timeout * 1000); // convert milliseconds -> microseconds + timeout = 0; + } + else { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "sleeping %d millieconds\n", sleep_ms); + switch_sleep(sleep_ms * 1000); // convert milliseconds -> microseconds + timeout -= sleep_ms; + } + + // call recursively + // TODO: This is going to end up in a lot of recursion and + // maybe blow the stack .. rethink this approach + return openmrcp_session_wait_for_event(openmrcp_session, openmrcp_event, timeout); + + } + else { + // nothing in queue, no timeout left, return failure + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "nothing in queue, no timeout left, return failure\n"); + return MRCP_STATUS_FAILURE; + } + + } + else { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "popped event\n"); + // check if popped event matches the event we are looking for + if (*popped_event == openmrcp_event) { + // just what we were waiting for! + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "just what we were waiting for! rturn success\n"); + return MRCP_STATUS_SUCCESS; + } + else { + // nothing popped, but maybe there's other things in queue, or something will arrive + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "popped unexpected\n"); + if (!popped_event) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "popped NULL!!\n"); + } + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "popped: %d\n", *popped_event); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "popped name: %s\n", openmrcp_event_names[*popped_event]); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "popped [%s] but was expecting [%s], but maybe there's other things in queue, or something will arrive\n", openmrcp_event_names[*popped_event], openmrcp_event_names[openmrcp_event]); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "calling recursively\n"); + return openmrcp_session_wait_for_event(openmrcp_session, openmrcp_event, timeout); + } + + } + } + else { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "event queue is null\n"); + return MRCP_STATUS_FAILURE; + } + +} + + +static mrcp_status_t openmrcp_session_signal_event(openmrcp_session_t *openmrcp_session, openmrcp_event_t openmrcp_event) +{ + mrcp_status_t status = MRCP_STATUS_SUCCESS; + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Got event: %s\n", openmrcp_event_names[openmrcp_event]); + + + // allocate memory for event + openmrcp_event_t *event2queue = (openmrcp_event_t *) switch_core_alloc(openmrcp_session->pool, sizeof(openmrcp_event_t)); + *event2queue = openmrcp_event; + + // add it to queue + if (switch_queue_trypush(openmrcp_session->event_queue, (void *) event2queue)) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "could not push event to queue\n"); + status = SWITCH_STATUS_GENERR; + } + else { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "pushed event to queue: %s\n", openmrcp_event_names[*event2queue]); + } + + return status; +} + + +static mrcp_status_t openmrcp_on_session_initiate(mrcp_client_context_t *context, mrcp_session_t *session) +{ + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "openmrcp_on_session_initiate called\n"); + openmrcp_session_t *openmrcp_session = mrcp_client_context_session_object_get(session); + if(!openmrcp_session) { + return MRCP_STATUS_FAILURE; + } + return openmrcp_session_signal_event(openmrcp_session,OPENMRCP_EVENT_SESSION_INITIATE); +} + +static mrcp_status_t openmrcp_on_session_terminate(mrcp_client_context_t *context, mrcp_session_t *session) +{ + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "openmrcp_on_session_terminate called\n"); + openmrcp_session_t *openmrcp_session = mrcp_client_context_session_object_get(session); + if(!openmrcp_session) { + return MRCP_STATUS_FAILURE; + } + return openmrcp_session_signal_event(openmrcp_session,OPENMRCP_EVENT_SESSION_TERMINATE); +} + +static mrcp_status_t openmrcp_on_channel_add(mrcp_client_context_t *context, mrcp_session_t *session, mrcp_client_channel_t *channel) +{ + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "openmrcp_on_channel_add called\n"); + openmrcp_session_t *openmrcp_session = mrcp_client_context_session_object_get(session); + if(!openmrcp_session) { + return MRCP_STATUS_FAILURE; + } + return openmrcp_session_signal_event(openmrcp_session,OPENMRCP_EVENT_CHANNEL_CREATE); +} + +static mrcp_status_t openmrcp_on_channel_remove(mrcp_client_context_t *context, mrcp_session_t *session, mrcp_client_channel_t *channel) +{ + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "openmrcp_on_channel_remove called\n"); + openmrcp_session_t *openmrcp_session = mrcp_client_context_session_object_get(session); + if(!openmrcp_session) { + return MRCP_STATUS_FAILURE; + } + return openmrcp_session_signal_event(openmrcp_session,OPENMRCP_EVENT_CHANNEL_DESTROY); +} + +/** this is called by the mrcp core whenever an mrcp message is received from + the other side. */ +static mrcp_status_t openmrcp_on_channel_modify(mrcp_client_context_t *context, mrcp_session_t *session, mrcp_message_t *mrcp_message) +{ + + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "openmrcp_on_channel_modify called\n"); + openmrcp_session_t *openmrcp_session = mrcp_client_context_session_object_get(session); + if(!openmrcp_session) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "!openmrcp_session\n"); + return MRCP_STATUS_FAILURE; + } + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "openmrcp_session: %p\n", openmrcp_session); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "mrcp msg method name: %s\n", mrcp_message->start_line.method_name); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "mrcp msg body: %s\n", mrcp_message->body); + + if (!strcmp(mrcp_message->start_line.method_name,"RECOGNITION-COMPLETE")) { + openmrcp_session->mrcp_message_last_rcvd = mrcp_message; + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "setting FLAG_HAS_TEXT\n"); + switch_set_flag_locked(openmrcp_session, FLAG_HAS_TEXT); + } + else if (!strcmp(mrcp_message->start_line.method_name,"START-OF-SPEECH")) { + openmrcp_session->mrcp_message_last_rcvd = mrcp_message; + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "setting FLAG_BARGE\n"); + switch_set_flag_locked(openmrcp_session, FLAG_BARGE); + } + else { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "ignoring method: %s\n", mrcp_message->start_line.method_name); + } + + return openmrcp_session_signal_event(openmrcp_session,OPENMRCP_EVENT_CHANNEL_MODIFY); +} + + +/* this needs to be moved to a config file */ +static apr_status_t set_default_options(openmrcp_client_options_t *options) +{ + options->client_ip = NULL; + options->client_port = 0; + options->server_ip = NULL; + options->server_port = 0; + options->rtp_port_min = 0; + options->rtp_port_max = 0; + mrcp_logger.priority = MRCP_PRIO_INFO; + + if(options->client_ip == NULL) { + options->client_ip = "127.0.0.1"; + } + if(options->client_port == 0) { + options->client_port = 8062; + } + if(options->server_ip == NULL) { + options->server_ip = "127.0.0.1"; + } + if(options->server_port == 0) { + options->server_port = 8060; + } + if(options->rtp_port_min == 0) { + options->rtp_port_min = 4000; + } + if(options->rtp_port_max == 0) { + options->rtp_port_max = 6000; + } + + return APR_SUCCESS; +} + +/** + * Called back by openmcp client thread every time its ready for more audio + * Reads data that was put into a shared fifo queue upon receiving audio frames + * from asr_feed() + */ +static apr_status_t openmrcp_recognizer_read_frame(audio_source_t *source, media_frame_t *frame) +{ + openmrcp_session_t *openmrcp_session = source->object; + frame->type = MEDIA_FRAME_TYPE_NONE; + apr_status_t result; + media_frame_t *queue_frame = NULL; + + + /* pop next media frame data from incoming queue into frame */ + if(openmrcp_session->audio_input_queue) { + if (switch_queue_trypop(openmrcp_session->audio_input_queue, &queue_frame)) { + // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "could not pop from queue\n"); + result = MRCP_STATUS_FAILURE; + } + else { + frame->codec_frame.size = queue_frame->codec_frame.size; + frame->codec_frame.buffer = queue_frame->codec_frame.buffer; + frame->type = MEDIA_FRAME_TYPE_AUDIO; + result = APR_SUCCESS; + } + } + else { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "no audio_in queue\n"); + result = MRCP_STATUS_FAILURE; + } + + return result; +} + +/** Read in the grammar and construct an MRCP Recognize message that has + The grammar attached as the payload */ +static mrcp_status_t openmrcp_recog_start(mrcp_client_context_t *context, openmrcp_session_t *openmrcp_session, char *path) +{ + + mrcp_generic_header_t *generic_header; + apr_status_t rv; + apr_file_t *fp; + apr_pool_t *mp; + apr_finfo_t finfo; + char *buf1; + int bytes2read = 0; + + mrcp_message_t *mrcp_message = mrcp_client_context_message_get(context, openmrcp_session->client_session, openmrcp_session->channel, RECOGNIZER_RECOGNIZE); + + if(!mrcp_message) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not create mrcp msg\n"); + return MRCP_STATUS_FAILURE; + } + + /* open the file with the grammar and read into char* buffer */ + mp = mrcp_message->pool; + if ((rv = apr_file_open(&fp, path, APR_READ, APR_OS_DEFAULT, mp)) != APR_SUCCESS) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not read grammar\n"); + return -1; + } + rv = apr_file_info_get(&finfo, APR_FINFO_NORM, fp); + + buf1 = apr_palloc(mp, finfo.size); + bytes2read = finfo.size; + rv = apr_file_read(fp, buf1, &bytes2read); + generic_header = mrcp_generic_header_prepare(mrcp_message); + if(!generic_header) { + return MRCP_STATUS_FAILURE; + } + + generic_header->content_type = mrcp_str_pdup(mrcp_message->pool,"application/srgs+xml"); + mrcp_generic_header_property_add(mrcp_message,GENERIC_HEADER_CONTENT_TYPE); + mrcp_message->body = mrcp_str_pdup(mrcp_message->pool,buf1); + + /* send the MRCP RECOGNIZE message to MRCP server */ + return mrcp_client_context_channel_modify(context, openmrcp_session->client_session, mrcp_message); + +} + + +/** + * Freeswitch calls this from switch_ivr_detect_speech() and then adds a media + * bug to tap into the channel's audio, which will result in all data getting + * passed to asr_feed() and calls to asr_check_results() on each recevied frame. + * + * This code expects certain one-time initialization of the openmrcp client + * engine/systeme to have already taken place.function to open the asr interface + */ +static switch_status_t mrcprec_asr_open(switch_asr_handle_t *ah, char *codec, int rate, char *dest, switch_asr_flag_t *flags) +{ + mrcp_client_t *mrcp_client = mrcprec_module.mrcp_client; + mrcp_client_context_t *mrcp_client_context = mrcprec_module.mrcp_client_context ; + openmrcp_client_options_t *options; + openmrcp_session_t *openmrcp_session; + audio_source_t *source; + + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "asr_open called, codec: %s, rate: %d\n", codec, rate); + + /*! + sanity checking regarding codec/rate - NOTE: some of the other codecs and rates might + actually work, but since they have not been tested yet, they are currently not + allowed. forcing MRCP to use 20 as the CODEC_FRAME_TIME_BASE effectively ensures + that it matches with 16-bit audio at 8kz with 320 byte frames. in testing, leaving + CODEC_FRAME_TIME_BASE at 10 and using pop (instead of trypop) in + openmrcp_recognizer_read_frame() actually produces clean audio, however it causes + other problems as the full channel/session cleanup never completes in openmrcp, most + likely due to a thread being blocked on a pop call from audio queue. but with trypop + (to avoid the cleanup problem), it only produces clean audio when CODEC_FRAME_TIME_BASE + is set to 20. + */ + if (strcmp(codec,"L16")) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Sorry, only L16 codec supported\n"); + return SWITCH_STATUS_GENERR; + } + if (rate != 8000) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Sorry, only 8kz supported\n"); + return SWITCH_STATUS_GENERR; + } + if (CODEC_FRAME_TIME_BASE != 20) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "You must recompile openmrcp with #define CODEC_FRAME_TIME_BASE 20\n"); + return SWITCH_STATUS_GENERR; + } + else { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "CODEC_FRAME_TIME_BASE: %d\n", CODEC_FRAME_TIME_BASE); + } + + /* create session */ + openmrcp_session = openmrcp_session_create(); + if (!openmrcp_session) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "openmrcp_session creation FAILED\n"); + return SWITCH_STATUS_GENERR; + } + openmrcp_session->client_session = mrcp_client_context_session_create(mrcp_client_context,openmrcp_session); + if (!openmrcp_session->client_session) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "openmrcp_session creation FAILED\n"); + return SWITCH_STATUS_GENERR; + } + + /* create audio source */ + source = mrcp_palloc(openmrcp_session->pool,sizeof(audio_source_t)); + source->method_set = &audio_source_method_set; + source->object = openmrcp_session; + openmrcp_session->source = source; + + /** + * create a new fifo queue. incoming audio received from freeswitch + * will be put into this queue, and it will be later pulled out by + * the openmrcp client thread. + */ + if (switch_queue_create(&openmrcp_session->audio_input_queue, 10000, openmrcp_session->pool)) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "audio queue creation failed\n"); + return SWITCH_STATUS_MEMERR; + } + + openmrcp_session->flags = *flags; + switch_mutex_init(&openmrcp_session->flag_mutex, SWITCH_MUTEX_NESTED, openmrcp_session->pool); + + ah->private_info = openmrcp_session; + + return SWITCH_STATUS_SUCCESS; + +} + +/* function to load a grammar to the asr interface */ +static switch_status_t mrcprec_asr_load_grammar(switch_asr_handle_t *ah, char *grammar, char *path) +{ + /** Read grammar from path and create and send and MRCP RECOGNIZE msg + that has the grammar attached to body. + + TODO: - how does DEFINE-GRAMMAR fit into the picture here? (if at all) + */ + + openmrcp_session_t *openmrcp_session = (openmrcp_session_t *) ah->private_info; + mrcp_client_t *mrcp_client = mrcprec_module.mrcp_client; + mrcp_client_context_t *mrcp_client_context = mrcprec_module.mrcp_client_context; + audio_source_t *source = openmrcp_session->source; + + /* create recognizer channel, also starts outgoing rtp media */ + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "Loading grammar\n"); + + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Create Recognizer Channel\n"); + openmrcp_session->channel = mrcp_client_recognizer_channel_create(mrcp_client_context, openmrcp_session->client_session, source); + + if (!openmrcp_session->channel) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Failed to create recognizer channel\n"); + return SWITCH_STATUS_FALSE; + } + + /* wait for recognizer channel creation */ + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "WAITING FOR CHAN CREATE\n"); + if(openmrcp_session_wait_for_event(openmrcp_session, OPENMRCP_EVENT_CHANNEL_CREATE, OPENMRCP_WAIT_TIMEOUT) == MRCP_STATUS_SUCCESS) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Got channel creation event\n"); + + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "WAITING FOR NONE\n"); + if (openmrcp_session_wait_for_event(openmrcp_session,OPENMRCP_EVENT_NONE,200) == MRCP_STATUS_SUCCESS) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "GOT NONE EVENT\n"); + } + else { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "NEVER GOT NONE EVENT\n"); + } + + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Start Recognizer\n"); + openmrcp_recog_start(mrcp_client_context, openmrcp_session, path); + + } + else { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "never got channel creation event\n"); + } + + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "Finished loading grammar\n"); + return SWITCH_STATUS_SUCCESS; +} + + +/*! function to feed audio to the ASR*/ +static switch_status_t mrcprec_asr_feed(switch_asr_handle_t *ah, void *data, unsigned int len, switch_asr_flag_t *flags) +{ + + openmrcp_session_t *openmrcp_session = (openmrcp_session_t *) ah->private_info; + mrcp_client_t *mrcp_client = mrcprec_module.mrcp_client; + mrcp_client_context_t *mrcp_client_context = mrcprec_module.mrcp_client_context; + media_frame_t *media_frame; + switch_byte_t *buffer; + + /* create new media frame */ + media_frame = (media_frame_t *) switch_core_alloc(openmrcp_session->pool, sizeof(media_frame_t)); + if (!media_frame) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "media_frame creation failed\n"); + return SWITCH_STATUS_MEMERR; + } + + /** since *data buffer might get freed by caller (true or false?), allocate a + new buffer and copy *data into it **/ + buffer = (switch_byte_t *) switch_core_alloc(openmrcp_session->pool, sizeof(switch_byte_t)*len); + if (!buffer) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not allocate buffer\n"); + return SWITCH_STATUS_MEMERR; + } + buffer = memcpy(buffer, data, len); + + media_frame->codec_frame.buffer = buffer; + media_frame->codec_frame.size = len; + media_frame->type = MEDIA_FRAME_TYPE_AUDIO; + + /* push audio to queue */ + if (switch_queue_trypush(openmrcp_session->audio_input_queue, (void *) media_frame)) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "could not push audio to queue\n"); + return SWITCH_STATUS_GENERR; + } + + return SWITCH_STATUS_SUCCESS; +} + + +static switch_status_t mrcprec_asr_pause(switch_asr_handle_t *ah) +{ + openmrcp_session_t *openmrcp_session = (openmrcp_session_t *) ah->private_info; + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "mrcprec_asr_pause called\n"); + + return SWITCH_STATUS_SUCCESS; + +} + +static switch_status_t mrcprec_asr_resume(switch_asr_handle_t *ah) +{ + openmrcp_session_t *openmrcp_session = (openmrcp_session_t *) ah->private_info; + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "mrcprec_asr_resume called\n"); + + return SWITCH_STATUS_SUCCESS; +} + + +/*! function to unload a grammar to the asr interface */ +static switch_status_t mrcprec_asr_unload_grammar(switch_asr_handle_t *ah, char *grammar) +{ + openmrcp_session_t *openmrcp_session = (openmrcp_session_t *) ah->private_info; + + return SWITCH_STATUS_SUCCESS; +} + +/** + * Freeswitch calls this whenever the channel is hungup or the + * speech detection is stopped via a call to switch_ivr_stop_detect_speech() + */ +static switch_status_t mrcprec_asr_close(switch_asr_handle_t *ah, switch_asr_flag_t *flags) +{ + openmrcp_session_t *openmrcp_session = (openmrcp_session_t *) ah->private_info; + + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "mrcprec_asr_close()\n"); + + mrcp_client_t *mrcp_client = mrcprec_module.mrcp_client; + mrcp_client_context_t *context = mrcprec_module.mrcp_client_context; + + // TODO!! should we do a switch_pool_clear(switch_memory_pool_t *p) on the pool held + // by openmrcp_session? + + // destroy channel + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Going to DESTROY CHANNEL\n"); + mrcp_client_context_channel_destroy(context, openmrcp_session->client_session, 1); + if (openmrcp_session_wait_for_event(openmrcp_session,OPENMRCP_EVENT_CHANNEL_DESTROY,10000) == MRCP_STATUS_SUCCESS) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "OPENMRCP_EVENT_CHANNEL_DESTROY received\n"); + } + else { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "timed out waiting for OPENMRCP_EVENT_CHANNEL_DESTROY\n"); + } + + // terminate client session + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Going to TERMINATE SESSION\n"); + mrcp_client_context_session_terminate(context, openmrcp_session->client_session); + if (openmrcp_session_wait_for_event(openmrcp_session,OPENMRCP_EVENT_SESSION_TERMINATE,10000) == MRCP_STATUS_SUCCESS) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "OPENMRCP_EVENT_SESSION_TERMINATE recevied\n"); + } + else { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "timed out waiting for OPENMRCP_EVENT_SESSION_TERMINATE\n"); + } + + // destroy client session (NOTE: this sends a BYE to the other side) + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Going to DESTROY SESSION\n"); + mrcp_client_context_session_destroy(context, openmrcp_session->client_session); + + // destroys the openmrcp_session struct + openmrcp_session_destroy(openmrcp_session); + + switch_set_flag(ah, SWITCH_ASR_FLAG_CLOSED); + + return SWITCH_STATUS_SUCCESS; +} + + +/** + * Freeswitch calls this method from the speech_thread() thread body method + * in switch_ivr_async.c every time a new frame is received by the media bug + * attached to the audio channel. If this method returns SWITCH_STATUS_SUCCESS, + * then Freeswitch will call mrcprec_asr_get_results() to get the result value. + */ +static switch_status_t mrcprec_asr_check_results(switch_asr_handle_t *ah, switch_asr_flag_t *flags) +{ + openmrcp_session_t *openmrcp_session = (openmrcp_session_t *) ah->private_info; + + switch_status_t rv = (switch_test_flag(openmrcp_session, FLAG_HAS_TEXT) || switch_test_flag(openmrcp_session, FLAG_BARGE)) ? SWITCH_STATUS_SUCCESS : SWITCH_STATUS_FALSE; + + return rv; +} + +/*! This will be called after asr_check_results returns SUCCESS */ +static switch_status_t mrcprec_asr_get_results(switch_asr_handle_t *ah, char **xmlstr, switch_asr_flag_t *flags) +{ + openmrcp_session_t *openmrcp_session = (openmrcp_session_t *) ah->private_info; + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "mrcprec_asr_get_results called\n"); + switch_status_t ret = SWITCH_STATUS_SUCCESS; + + if (switch_test_flag(openmrcp_session, FLAG_BARGE)) { + switch_clear_flag_locked(openmrcp_session, FLAG_BARGE); + ret = SWITCH_STATUS_BREAK; + } + + if (switch_test_flag(openmrcp_session, FLAG_HAS_TEXT)) { + /*! + we have to extract the XML but stripping off the <?xml version="1.0"?> + header. the body looks like: + + Completion-Cause:001 no-match + Content-Type: application/nlsml+xml + Content-Length: 260 + + <?xml version="1.0"?> + <result xmlns="http://www.ietf.org/xml/ns/mrcpv2" xmlns:ex="http://www.example.com/example" score="100" grammar="session:request1@form-level.store"> + <interpretation> + <input mode="speech">open a</input> + </interpretation> + </result> + */ + + char *marker = "?>"; // FIXME -- lame and brittle way of doing this. use regex or better. + char *position = strstr(openmrcp_session->mrcp_message_last_rcvd->body, marker); + if (!position) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Bad result received from mrcp server: %s", openmrcp_session->mrcp_message_last_rcvd->body); + ret = SWITCH_STATUS_FALSE; + } + else { + position += strlen(marker); + *xmlstr = strdup(position); + } + + // since we are returning our result here, future calls to check_results + // should return False + switch_clear_flag_locked(openmrcp_session, FLAG_HAS_TEXT); + ret = SWITCH_STATUS_SUCCESS; + } + return ret; +} + +static switch_asr_interface_t mrcprec_asr_interface = { + /*.interface_name*/ "mrcprec", + /*.asr_open*/ mrcprec_asr_open, + /*.asr_load_grammar*/ mrcprec_asr_load_grammar, + /*.asr_unload_grammar*/ mrcprec_asr_unload_grammar, + /*.asr_close*/ mrcprec_asr_close, + /*.asr_feed*/ mrcprec_asr_feed, + /*.asr_resume*/ mrcprec_asr_resume, + /*.asr_pause*/ mrcprec_asr_pause, + /*.asr_check_results*/ mrcprec_asr_check_results, + /*.asr_get_results*/ mrcprec_asr_get_results +}; + +static switch_loadable_module_interface_t mrcprec_module_interface = { + /*.module_name */ modname, + /*.endpoint_interface */ NULL, + /*.timer_interface */ NULL, + /*.dialplan_interface */ NULL, + /*.codec_interface */ NULL, + /*.application_interface */ NULL, + /*.api_interface */ NULL, + /*.file_interface */ NULL, + /*.speech_interface */ NULL, + /*.directory_interface */ NULL, + /*.chat_interface */ NULL, + /*.say_interface */ NULL, + /*.asr_interface */ &mrcprec_asr_interface +}; + + +static switch_status_t mrcp_init() +{ + /*! + Perform one-time initialization of client library + */ + + mrcp_mem_pool_t *pool; + mrcp_client_event_handler_t *client_event_handler; + mrcp_client_t *mrcp_client; + mrcp_client_context_t *mrcp_client_context; + openmrcp_client_options_t *options; + + pool = mrcp_global_pool_get(); + options = mrcp_palloc(pool,sizeof(openmrcp_client_options_t)); + client_event_handler = mrcp_palloc(pool,sizeof(mrcp_client_event_handler_t)); + set_default_options(options); + + client_event_handler->on_session_initiate = openmrcp_on_session_initiate; + client_event_handler->on_session_terminate = openmrcp_on_session_terminate; + client_event_handler->on_channel_add = openmrcp_on_channel_add; + client_event_handler->on_channel_remove = openmrcp_on_channel_remove; + client_event_handler->on_channel_modify = openmrcp_on_channel_modify; + + // create client context, which to must be passed to client engine + mrcp_client_context = mrcp_client_context_create(&mrcprec_module,client_event_handler); + if(!mrcp_client_context) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "mrcp_client_context creation failed\n"); + return SWITCH_STATUS_GENERR; + } + mrcprec_module.mrcp_client_context = mrcp_client_context; + + // this basically starts a thread that pulls events from the event queue + // and handles them + mrcp_client = openmrcp_client_start(options,mrcp_client_context); + if(!mrcp_client) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "openmrcp_client_start FAILED\n"); + mrcp_client_context_destroy(mrcp_client_context); + return SWITCH_STATUS_GENERR; + } + + mrcprec_module.mrcp_client = mrcp_client; + + return SWITCH_STATUS_SUCCESS; +} + +SWITCH_MODULE_LOAD_FUNCTION(mod_mrcprec_load) +{ + + /* connect my internal structure to the blank pointer passed to me */ + *module_interface = &mrcprec_module_interface; + + /* initialize openmrcp */ + mrcp_global_init(); + mrcp_init(); + + /* indicate that the module should continue to be loaded */ + return SWITCH_STATUS_SUCCESS; +} + +SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_mrcprec_shutdown) +{ + return SWITCH_STATUS_UNLOAD; +}

1 0

[Freeswitch-branches] [commit] r5576 - in freeswitch/branches/greenlizard/src: . include mod/languages/mod_python
by Freeswitch SVN 07 Aug '07

07 Aug '07

Author: greenlizard Date: Mon Aug 6 21:33:15 2007 New Revision: 5576 Modified: freeswitch/branches/greenlizard/src/include/switch_cpp.h freeswitch/branches/greenlizard/src/mod/languages/mod_python/freeswitch.py freeswitch/branches/greenlizard/src/mod/languages/mod_python/freeswitch_python.cpp freeswitch/branches/greenlizard/src/mod/languages/mod_python/freeswitch_python.h freeswitch/branches/greenlizard/src/mod/languages/mod_python/mod_python_wrap.cpp freeswitch/branches/greenlizard/src/switch_cpp.cpp Log: 1. add collectInput method. 2. modify the dtmf callback handlers to better deal with event inputs, as opposed to dtmf inputs. currently returns a dictionary where one key contains the body, and another contains the headers as a subdictionary. Modified: freeswitch/branches/greenlizard/src/include/switch_cpp.h ============================================================================== --- freeswitch/branches/greenlizard/src/include/switch_cpp.h (original) +++ freeswitch/branches/greenlizard/src/include/switch_cpp.h Mon Aug 6 21:33:15 2007 @@ -158,7 +158,7 @@ * * The DTMF callback function will be set and persist * for the life of the session, and be called when a dtmf - * is pressed by user during playFile(), streamfile(), and + * is pressed by user during streamfile(), collectDigits(), and * certain other methods are executing. * */ @@ -168,6 +168,19 @@ int speak(char *text); void set_tts_parms(char *tts_name, char *voice_name); + /** + * For timeout milliseconds, call the dtmf function set previously + * by setDTMFCallback whenever a dtmf or event is received + */ + int collectDigits(int timeout); + + /** + * Collect up to maxdigits digits worth of digits + * and store them in dtmf_buf. In the case of mod_python, the + * dtmf_buf parameter is configured to act as a _return_ value, + * (see mod_python.i). This does NOT call any callbacks upon + * receiving dtmf digits. For that, use collectDigits. + */ int getDigits(char *dtmf_buf, int buflen, int maxdigits, Modified: freeswitch/branches/greenlizard/src/mod/languages/mod_python/freeswitch.py ============================================================================== --- freeswitch/branches/greenlizard/src/mod/languages/mod_python/freeswitch.py (original) +++ freeswitch/branches/greenlizard/src/mod/languages/mod_python/freeswitch.py Mon Aug 6 21:33:15 2007 @@ -118,6 +118,7 @@ def setDTMFCallback(*args): return _freeswitch.CoreSession_setDTMFCallback(*args) def speak(*args): return _freeswitch.CoreSession_speak(*args) def set_tts_parms(*args): return _freeswitch.CoreSession_set_tts_parms(*args) + def collectDigits(*args): return _freeswitch.CoreSession_collectDigits(*args) def getDigits(*args): return _freeswitch.CoreSession_getDigits(*args) def transfer(*args): return _freeswitch.CoreSession_transfer(*args) def playAndGetDigits(*args): return _freeswitch.CoreSession_playAndGetDigits(*args) Modified: freeswitch/branches/greenlizard/src/mod/languages/mod_python/freeswitch_python.cpp ============================================================================== --- freeswitch/branches/greenlizard/src/mod/languages/mod_python/freeswitch_python.cpp (original) +++ freeswitch/branches/greenlizard/src/mod/languages/mod_python/freeswitch_python.cpp Mon Aug 6 21:33:15 2007 @@ -105,10 +105,12 @@ } switch_status_t PySession::run_dtmf_callback(void *input, - switch_input_type_t itype) { + switch_input_type_t itype) { PyObject *func, *arglist; PyObject *pyresult; + PyObject* headerdict; + char *resultStr; char *funcargs; switch_file_handle_t *fh = NULL; @@ -137,7 +139,47 @@ funcargs = (char *) cb_state.funcargs; - arglist = Py_BuildValue("(sis)", input, itype, funcargs); + if (itype == SWITCH_INPUT_TYPE_DTMF) { + + arglist = Py_BuildValue("(sis)", input, itype, funcargs); + } + else if (itype == SWITCH_INPUT_TYPE_EVENT) { + // DUNNO if this is correct in the case we have an event + // will be of type switch_event_t *event; + // http://www.freeswitch.org/docs/structswitch__event.html + switch_event_t *event = (switch_event_t *) input; + arglist = Py_BuildValue("({s:s}is)", + "body", event->body, + itype, + funcargs); + + // build a dictionary with all the headers + + switch_event_header_t *hp; + headerdict = PyDict_New(); + for (hp = event->headers; hp; hp = hp->next) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Adding event header to result"); + PyDict_SetItem(headerdict, + Py_BuildValue("s", hp->name), + Py_BuildValue("s", hp->value)); + + } + + // add it to the main event dictionary (first arg in list) + // under key 'headers' + PyObject *dict = PyTuple_GetItem(arglist, 0); + PyDict_SetItemString(dict, + "headers", + headerdict); + + //arglist = Py_BuildValue("(sis)", event->body, itype, funcargs); + } + else { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Unknown input type: %d\n", itype); + return SWITCH_STATUS_FALSE; + } + + if (!arglist) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "error building arglist"); return SWITCH_STATUS_FALSE; @@ -150,13 +192,15 @@ Py_XDECREF(arglist); // Trash arglist + Py_XDECREF(headerdict); + if (pyresult && pyresult != Py_None) { resultStr = (char *) PyString_AsString(pyresult); switch_status_t cbresult = process_callback_result(resultStr, &cb_state, session); return cbresult; } else { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error calling python callback\n"); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Python callback\n returned None"); PyErr_Print(); PyErr_Clear(); } Modified: freeswitch/branches/greenlizard/src/mod/languages/mod_python/freeswitch_python.h ============================================================================== --- freeswitch/branches/greenlizard/src/mod/languages/mod_python/freeswitch_python.h (original) +++ freeswitch/branches/greenlizard/src/mod/languages/mod_python/freeswitch_python.h Mon Aug 6 21:33:15 2007 @@ -40,8 +40,19 @@ bool begin_allow_threads(); bool end_allow_threads(); + /** + * Run DTMF callback + * + * A static method in CoreSession is the first thing called + * upon receving a dtmf/event callback from fs engine, and then + * it gets the PySession instance and calls this method with + * dtmf/event object. + * + * @param input - a dtmf char buffer, or an event 'object' (not sure..) + * @param itype - a SWITCH_INPUT_TYPE_DTMF or a SWITCH_INPUT_TYPE_EVENT + */ switch_status_t run_dtmf_callback(void *input, - switch_input_type_t itype); + switch_input_type_t itype); }; Modified: freeswitch/branches/greenlizard/src/mod/languages/mod_python/mod_python_wrap.cpp ============================================================================== --- freeswitch/branches/greenlizard/src/mod/languages/mod_python/mod_python_wrap.cpp (original) +++ freeswitch/branches/greenlizard/src/mod/languages/mod_python/mod_python_wrap.cpp Mon Aug 6 21:33:15 2007 @@ -4080,6 +4080,37 @@ } +SWIGINTERN PyObject *_wrap_CoreSession_collectDigits(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *resultobj = 0; + CoreSession *arg1 = (CoreSession *) 0 ; + int arg2 ; + int result; + void *argp1 = 0 ; + int res1 = 0 ; + int val2 ; + int ecode2 = 0 ; + PyObject * obj0 = 0 ; + PyObject * obj1 = 0 ; + + if (!PyArg_ParseTuple(args,(char *)"OO:CoreSession_collectDigits",&obj0,&obj1)) SWIG_fail; + res1 = SWIG_ConvertPtr(obj0, &argp1,SWIGTYPE_p_CoreSession, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "CoreSession_collectDigits" "', argument " "1"" of type '" "CoreSession *""'"); + } + arg1 = reinterpret_cast< CoreSession * >(argp1); + ecode2 = SWIG_AsVal_int(obj1, &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "CoreSession_collectDigits" "', argument " "2"" of type '" "int""'"); + } + arg2 = static_cast< int >(val2); + result = (int)(arg1)->collectDigits(arg2); + resultobj = SWIG_From_int(static_cast< int >(result)); + return resultobj; +fail: + return NULL; +} + + SWIGINTERN PyObject *_wrap_CoreSession_getDigits(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { PyObject *resultobj = 0; CoreSession *arg1 = (CoreSession *) 0 ; @@ -5228,6 +5259,7 @@ { (char *)"CoreSession_setDTMFCallback", _wrap_CoreSession_setDTMFCallback, METH_VARARGS, NULL}, { (char *)"CoreSession_speak", _wrap_CoreSession_speak, METH_VARARGS, NULL}, { (char *)"CoreSession_set_tts_parms", _wrap_CoreSession_set_tts_parms, METH_VARARGS, NULL}, + { (char *)"CoreSession_collectDigits", _wrap_CoreSession_collectDigits, METH_VARARGS, NULL}, { (char *)"CoreSession_getDigits", _wrap_CoreSession_getDigits, METH_VARARGS, NULL}, { (char *)"CoreSession_transfer", _wrap_CoreSession_transfer, METH_VARARGS, NULL}, { (char *)"CoreSession_playAndGetDigits", _wrap_CoreSession_playAndGetDigits, METH_VARARGS, NULL}, Modified: freeswitch/branches/greenlizard/src/switch_cpp.cpp ============================================================================== --- freeswitch/branches/greenlizard/src/switch_cpp.cpp (original) +++ freeswitch/branches/greenlizard/src/switch_cpp.cpp Mon Aug 6 21:33:15 2007 @@ -92,7 +92,7 @@ const switch_application_interface_t *application_interface; sanity_check_noreturn; - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "CoreSession::execute. data:%s\n", data); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "CoreSession::execute. app: %s data:%s\n", app, data); if ((application_interface = switch_loadable_module_get_application_interface(app))) { begin_allow_threads(); switch_core_session_exec(session, application_interface, data); @@ -158,6 +158,15 @@ voice_name = strdup(voice_name_p); } + + +int CoreSession::collectDigits(int timeout) { + sanity_check(-1); + begin_allow_threads(); + switch_ivr_collect_digits_callback(session, ap, timeout); + end_allow_threads(); +} + int CoreSession::getDigits(char *dtmf_buf, int buflen, int maxdigits, @@ -214,7 +223,8 @@ terminators, audio_files, bad_input_audio_files, - dtmf_buf, 128, + dtmf_buf, + 128, digits_regex); switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "playAndGetDigits dtmf_buf: %s\n", dtmf_buf); @@ -522,7 +532,7 @@ CoreSession *coresession = NULL; switch_status_t result; - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "dtmf_callback called\n"); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "dtmf_callback called\n"); fflush(stdout); channel = switch_core_session_get_channel(session_cb);

1 0