[Freeswitch-svn] [commit] r5582 - freeswitch/branches/greenlizard/src/mod/asr_tts/mod_mrcprec

Freeswitch SVN greenlizard at freeswitch.org
Thu Aug 9 17:24:12 EDT 2007


Author: greenlizard
Date: Thu Aug  9 17:24:12 2007
New Revision: 5582

Added:
   freeswitch/branches/greenlizard/src/mod/asr_tts/mod_mrcprec/
   freeswitch/branches/greenlizard/src/mod/asr_tts/mod_mrcprec/Makefile
   freeswitch/branches/greenlizard/src/mod/asr_tts/mod_mrcprec/mod_mrcprec.c

Log:
rough (but working) version of a module which connects to a speech recognition server via openmrcp client lib

Added: freeswitch/branches/greenlizard/src/mod/asr_tts/mod_mrcprec/Makefile
==============================================================================
--- (empty file)
+++ freeswitch/branches/greenlizard/src/mod/asr_tts/mod_mrcprec/Makefile	Thu Aug  9 17:24:12 2007
@@ -0,0 +1,27 @@
+# define these targets in your makefile if you wish
+# local_all local_depend local_clean depend_install local_install local_distclean local_extraclean:
+
+# and define these variables to impact your build
+
+# TODO
+# - needs to compile/link against the freeswitch sofia libs
+# - tie up these includes somehow so there isn't so many
+# - ditto for the .a library files
+
+OPENMRCP_DIR=/usr/src/openmrcp_stable  # FIXME!!!
+
+OPENMRCP_INCLUDE=-I$(OPENMRCP_DIR)/mrcpcore/client/include/ -I$(OPENMRCP_DIR)/platform/openmrcpclient/include/ -I$(OPENMRCP_DIR)/mediaframe/include/ -I$(OPENMRCP_DIR)/mrcpcore/engine/include/ -I$(OPENMRCP_DIR)/mrcpcore/include/ -I$(OPENMRCP_DIR)/mrcpcore/parser/include/ -I$(OPENMRCP_DIR)/mrcpcore/server/include/ -I$(OPENMRCP_DIR)/mrcpcore/media/include/ -I$(OPENMRCP_DIR)/mrcpcore/util/include -I$(OPENMRCP_DIR)/mrcpcore/resource/include/
+
+OPENMRCP_LIBS=$(OPENMRCP_DIR)/platform/openmrcpclient/.libs/libopenmrcpclient.a $(OPENMRCP_DIR)/platform/openmrcpserver/.libs/libopenmrcpserver.a $(OPENMRCP_DIR)/mrcpcore/client/.libs/libmrcpclient.a $(OPENMRCP_DIR)/mediaframe/.libs/libmediaframe.a $(OPENMRCP_DIR)/mrcpcore/engine/.libs/libmrcpengine.a $(OPENMRCP_DIR)/mrcpcore/parser/.libs/libmrcpparser.a $(OPENMRCP_DIR)/mrcpcore/server/.libs/libmrcpserver.a $(OPENMRCP_DIR)/mrcpcore/media/.libs/libmrcpmedia.a $(OPENMRCP_DIR)/mrcpcore/util/.libs/libmrcputil.a $(OPENMRCP_DIR)/mrcpcore/util/.libs/libmrcputil.a $(OPENMRCP_DIR)/mrcpcore/resource/.libs/libmrcpresource.a $(OPENMRCP_DIR)/module/plugin/demo/.libs/libdemoplugin.a $(OPENMRCP_DIR)/module/mrcpv2agent/.libs/libmrcpv2agent.a $(OPENMRCP_DIR)/module/mrcplivemedia/.libs/libmrcplivemedia.a $(OPENMRCP_DIR)/module/mrcpsofiasip/.libs/libmrcpsofiasip.a $(OPENMRCP_DIR)/mrcpcore/.libs/libmrcpcore.a 
+
+MRCP_APR_INCLUDES =  -I../../../../libs/apr/include
+MRCP_APU_INCLUDES =  -I../../../../libs/apr-util/include
+MRCP_SOFIA_INCLUDES = -I/usr/local/include/sofia-sip-1.12  # FIXME!!
+
+LOCAL_CFLAGS= -Wno-unused
+LOCAL_CFLAGS+= -Wno-comment -Wno-sign-compare $(OPENMRCP_INCLUDE) $(MRCP_APR_INCLUDES) $(MRCP_APU_INCLUDES) $(MRCP_SOFIA_INCLUDES)
+
+LOCAL_LDFLAGS=$(OPENMRCP_LIBS)
+VERBOSE=TRUE
+include ../../../../build/modmake.rules
+

Added: freeswitch/branches/greenlizard/src/mod/asr_tts/mod_mrcprec/mod_mrcprec.c
==============================================================================
--- (empty file)
+++ freeswitch/branches/greenlizard/src/mod/asr_tts/mod_mrcprec/mod_mrcprec.c	Thu Aug  9 17:24:12 2007
@@ -0,0 +1,875 @@
+/* 
+ * FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application
+ * Copyright (C) 2005/2006, Anthony Minessale II <anthmct at yahoo.com>
+ *
+ * Version: MPL 1.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application
+ *
+ * The Initial Developer of the Original Code is
+ * Anthony Minessale II <anthmct at yahoo.com>
+ * Portions created by the Initial Developer are Copyright (C)
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ * 
+ * Traun Leyden <tleyden at branchcut.com>
+ *
+ * Module which acts as an MRCP client to an MRCP speech recognition
+ * server.  In other words it bridges freeswitch to an external speech
+ * recognition system.  
+ *
+ * Uses OpenMrcp (http://wiki.freeswitch.org/wiki/OpenMRCP) as the 
+ * the client library.
+ * 
+ */
+#ifdef __ICC
+#pragma warning (disable:188)
+#endif
+
+
+#include "openmrcp_client.h"
+#include "mrcp_client_context.h"
+#include "mrcp_recognizer.h"
+#include "mrcp_generic_header.h"
+#include "rtp_session.h"
+#include "mrcp_client.h"
+#include "mrcp_client_context.h"
+#include "mrcp_client_defs.h"
+#include "mrcp_client_session.h"
+#include "mrcp_client_resource.h"
+#include "mrcp_client_signaling_agent.h"
+#include "mrcp_media_agent.h"
+#include "mrcp_resource.h"
+#include "mrcp_consumer_task.h"
+#include <apr_general.h>
+#include <apr_file_io.h>
+#include <apr_thread_proc.h>
+#include <apr_thread_cond.h>
+#include <apr_strings.h>
+
+#include <switch.h>
+	
+#define OPENMRCP_WAIT_TIMEOUT 5000
+
+SWITCH_MODULE_LOAD_FUNCTION(mod_mrcprec_load);
+SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_mrcprec_shutdown);
+SWITCH_MODULE_DEFINITION(mod_mrcprec, mod_mrcprec_load, 
+						 mod_mrcprec_shutdown, NULL);
+
+typedef enum {
+	OPENMRCP_EVENT_NONE,
+	OPENMRCP_EVENT_SESSION_INITIATE,
+	OPENMRCP_EVENT_SESSION_TERMINATE,
+	OPENMRCP_EVENT_CHANNEL_CREATE,
+	OPENMRCP_EVENT_CHANNEL_DESTROY,
+	OPENMRCP_EVENT_CHANNEL_MODIFY
+} openmrcp_event_t;
+
+static const char *openmrcp_event_names[] = {
+	"NONE",
+	"SESSION_INITIATE",
+	"SESSION_TERMINATE",
+	"CHANNEL_CREATE",
+	"CHANNEL_DESTROY",
+	"CHANNEL_MODIFY",
+};
+
+// TODO: rename this to mrcprec_session_t 
+typedef struct openmrcp_session_t openmrcp_session_t;
+struct openmrcp_session_t {
+	mrcp_session_t        *client_session;
+	mrcp_client_channel_t *channel;
+	switch_queue_t        *audio_input_queue;
+	switch_queue_t        *event_queue;
+	mrcp_message_t        *mrcp_message_last_rcvd;
+	audio_source_t        *source;
+	apr_pool_t            *pool;
+	apr_thread_cond_t     *wait_object;
+	apr_thread_mutex_t    *wait_object_mutex;
+	openmrcp_event_t       wait_event;
+	uint32_t 			   flags;
+	switch_mutex_t        *flag_mutex;
+};
+
+static apr_status_t openmrcp_recognizer_read_frame(audio_source_t *source, media_frame_t *frame);
+
+static const audio_source_method_set_t audio_source_method_set = {
+	NULL,
+	NULL,
+	NULL,
+	openmrcp_recognizer_read_frame
+};
+
+typedef enum {
+	FLAG_HAS_TEXT = (1 << 0),
+	FLAG_BARGE = (1 << 1),
+	FLAG_READY = (1 << 2)
+} mrcp_flag_t;
+
+typedef struct {
+	mrcp_client_t *mrcp_client;
+	mrcp_client_context_t *mrcp_client_context;
+} mrcprec_module_t;
+
+static mrcprec_module_t mrcprec_module;
+
+static void log_callback(const char* message, void* userdata)
+{
+	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "%s\n", message);
+}
+
+static const char *state_names[] = {
+	"NOT_READY",
+	"READY",
+	"BARGE_IN",
+	"END_SPEECH",
+	"STOPPED",
+	"BARGE_IN_TIMEOUT",
+	"END_SPEECH_TIMEOUT",
+	"BEEP"
+};
+
+static openmrcp_session_t* openmrcp_session_create()
+{
+	openmrcp_session_t *openmrcp_session;
+	apr_pool_t *session_pool;
+
+	if(apr_pool_create(&session_pool,NULL) != APR_SUCCESS) {
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "failed to create session_pool\n");
+		return NULL;
+	}
+
+	openmrcp_session = apr_palloc(session_pool,sizeof(openmrcp_session_t));
+	openmrcp_session->pool = session_pool;
+	openmrcp_session->client_session = NULL;
+	openmrcp_session->channel = NULL;
+	openmrcp_session->audio_input_queue = NULL;
+	openmrcp_session->event_queue = NULL;
+	openmrcp_session->wait_event = OPENMRCP_EVENT_NONE;
+
+	apr_thread_mutex_create(&openmrcp_session->wait_object_mutex, APR_THREAD_MUTEX_UNNESTED, openmrcp_session->pool);
+
+	apr_thread_cond_create(&openmrcp_session->wait_object, openmrcp_session->pool);
+
+
+	/* create an event queue */
+	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO,  "creating event queue\n");
+	if (switch_queue_create(&openmrcp_session->event_queue, 1000, openmrcp_session->pool)) {
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR,  "event queue creation failed\n");
+	}
+
+	return openmrcp_session;
+}
+
+static mrcp_status_t openmrcp_session_destroy(openmrcp_session_t *openmrcp_session)
+{
+	if(!openmrcp_session) {
+		return MRCP_STATUS_FAILURE;
+	}
+
+	if(openmrcp_session->pool) {
+		apr_thread_cond_destroy(openmrcp_session->wait_object);
+		apr_thread_mutex_destroy(openmrcp_session->wait_object_mutex);
+		apr_pool_destroy(openmrcp_session->pool);
+		openmrcp_session->pool = NULL;
+	}
+	return MRCP_STATUS_SUCCESS;
+}
+
+
+static mrcp_status_t openmrcp_session_wait_for_event(openmrcp_session_t *openmrcp_session, openmrcp_event_t openmrcp_event, size_t timeout)
+{
+	mrcp_status_t status = MRCP_STATUS_SUCCESS;
+	openmrcp_event_t *popped_event = NULL;
+	size_t sleep_ms = 100;
+
+	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "openmrcp_session_wait_for_event()\n");
+
+	if(openmrcp_session->event_queue) {
+		if (switch_queue_trypop(openmrcp_session->event_queue, &popped_event)) {
+			// most likely this failed because queue is empty.  how about we sleep for
+			// timeout seconds and try again?
+			switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "did not pop anything from event queue\n");
+
+			if (timeout > 0) {
+
+				if (timeout < sleep_ms) {
+					switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "sleeping %d millieconds\n", timeout);
+					switch_sleep(timeout * 1000);  // convert milliseconds -> microseconds
+					timeout = 0;  
+				}
+				else {
+					switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "sleeping %d millieconds\n", sleep_ms);
+					switch_sleep(sleep_ms * 1000);  // convert milliseconds -> microseconds
+					timeout -= sleep_ms;  
+				}
+
+				// call recursively
+				// TODO: This is going to end up in a lot of recursion and
+				// maybe blow the stack .. rethink this approach
+				return openmrcp_session_wait_for_event(openmrcp_session, openmrcp_event, timeout);
+
+			}
+			else {
+				// nothing in queue, no timeout left, return failure
+				switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "nothing in queue, no timeout left, return failure\n");
+				return MRCP_STATUS_FAILURE;
+			}
+
+		}
+		else {
+			switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "popped event\n");
+			// check if popped event matches the event we are looking for
+			if (*popped_event == openmrcp_event) {
+				// just what we were waiting for!  
+				switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "just what we were waiting for! rturn success\n");
+				return MRCP_STATUS_SUCCESS;
+			}
+			else {
+				// nothing popped, but maybe there's other things in queue, or something will arrive
+				switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "popped unexpected\n");
+				if (!popped_event) {
+					switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "popped NULL!!\n");
+				}
+				switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "popped: %d\n", *popped_event);
+				switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "popped name: %s\n", openmrcp_event_names[*popped_event]);
+				switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "popped [%s] but was expecting [%s], but maybe there's other things in queue, or something will arrive\n", openmrcp_event_names[*popped_event], openmrcp_event_names[openmrcp_event]);
+				switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "calling recursively\n");
+				return openmrcp_session_wait_for_event(openmrcp_session, openmrcp_event, timeout);
+			}
+			
+		}
+	}
+	else {
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "event queue is null\n");
+		return MRCP_STATUS_FAILURE;
+	}
+
+}
+
+
+static mrcp_status_t openmrcp_session_signal_event(openmrcp_session_t *openmrcp_session, openmrcp_event_t openmrcp_event)
+{
+	mrcp_status_t status = MRCP_STATUS_SUCCESS;
+	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Got event: %s\n", openmrcp_event_names[openmrcp_event]);
+
+
+	// allocate memory for event
+	openmrcp_event_t *event2queue = (openmrcp_event_t *) switch_core_alloc(openmrcp_session->pool, sizeof(openmrcp_event_t));
+	*event2queue = openmrcp_event;
+
+	// add it to queue
+	if (switch_queue_trypush(openmrcp_session->event_queue, (void *) event2queue)) {
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "could not push event to queue\n");
+		status = SWITCH_STATUS_GENERR;
+	}
+	else {
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "pushed event to queue: %s\n", openmrcp_event_names[*event2queue]);
+	}
+
+	return status;
+}
+
+
+static mrcp_status_t openmrcp_on_session_initiate(mrcp_client_context_t *context, mrcp_session_t *session)
+{
+	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "openmrcp_on_session_initiate called\n");
+	openmrcp_session_t *openmrcp_session = mrcp_client_context_session_object_get(session);
+	if(!openmrcp_session) {
+		return MRCP_STATUS_FAILURE;
+	}
+	return openmrcp_session_signal_event(openmrcp_session,OPENMRCP_EVENT_SESSION_INITIATE);
+}
+
+static mrcp_status_t openmrcp_on_session_terminate(mrcp_client_context_t *context, mrcp_session_t *session)
+{
+	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "openmrcp_on_session_terminate called\n");
+	openmrcp_session_t *openmrcp_session = mrcp_client_context_session_object_get(session);
+	if(!openmrcp_session) {
+		return MRCP_STATUS_FAILURE;
+	}
+	return openmrcp_session_signal_event(openmrcp_session,OPENMRCP_EVENT_SESSION_TERMINATE);
+}
+
+static mrcp_status_t openmrcp_on_channel_add(mrcp_client_context_t *context, mrcp_session_t *session, mrcp_client_channel_t *channel)
+{
+	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "openmrcp_on_channel_add called\n");
+	openmrcp_session_t *openmrcp_session = mrcp_client_context_session_object_get(session);
+	if(!openmrcp_session) {
+		return MRCP_STATUS_FAILURE;
+	}
+	return openmrcp_session_signal_event(openmrcp_session,OPENMRCP_EVENT_CHANNEL_CREATE);
+}
+
+static mrcp_status_t openmrcp_on_channel_remove(mrcp_client_context_t *context, mrcp_session_t *session, mrcp_client_channel_t *channel)
+{
+	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "openmrcp_on_channel_remove called\n");
+	openmrcp_session_t *openmrcp_session = mrcp_client_context_session_object_get(session);
+	if(!openmrcp_session) {
+		return MRCP_STATUS_FAILURE;
+	}
+	return openmrcp_session_signal_event(openmrcp_session,OPENMRCP_EVENT_CHANNEL_DESTROY);
+}
+
+/** this is called by the mrcp core whenever an mrcp message is received from
+    the other side. */
+static mrcp_status_t openmrcp_on_channel_modify(mrcp_client_context_t *context, mrcp_session_t *session, mrcp_message_t *mrcp_message)
+{
+
+	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "openmrcp_on_channel_modify called\n");
+	openmrcp_session_t *openmrcp_session = mrcp_client_context_session_object_get(session);
+	if(!openmrcp_session) {
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "!openmrcp_session\n");
+		return MRCP_STATUS_FAILURE;
+	}
+	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "openmrcp_session: %p\n", openmrcp_session);
+	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "mrcp msg method name: %s\n", mrcp_message->start_line.method_name);
+	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "mrcp msg body: %s\n", mrcp_message->body);
+
+	if (!strcmp(mrcp_message->start_line.method_name,"RECOGNITION-COMPLETE")) {
+		openmrcp_session->mrcp_message_last_rcvd = mrcp_message;
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "setting FLAG_HAS_TEXT\n");
+		switch_set_flag_locked(openmrcp_session, FLAG_HAS_TEXT);
+	}
+	else if (!strcmp(mrcp_message->start_line.method_name,"START-OF-SPEECH")) {
+		openmrcp_session->mrcp_message_last_rcvd = mrcp_message;
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "setting FLAG_BARGE\n");
+		switch_set_flag_locked(openmrcp_session, FLAG_BARGE);
+	}
+	else {
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "ignoring method: %s\n", mrcp_message->start_line.method_name);
+	}
+		
+	return openmrcp_session_signal_event(openmrcp_session,OPENMRCP_EVENT_CHANNEL_MODIFY);
+}
+
+
+/* this needs to be moved to a config file */
+static apr_status_t set_default_options(openmrcp_client_options_t *options)
+{
+	options->client_ip = NULL;
+	options->client_port = 0;
+	options->server_ip = NULL;
+	options->server_port = 0;
+	options->rtp_port_min = 0;
+	options->rtp_port_max = 0;
+	mrcp_logger.priority = MRCP_PRIO_INFO;
+
+	if(options->client_ip == NULL) {
+		options->client_ip = "127.0.0.1";
+	}
+	if(options->client_port == 0) {
+		options->client_port = 8062;
+	}
+	if(options->server_ip == NULL) {
+		options->server_ip = "127.0.0.1";
+	}
+	if(options->server_port == 0) {
+		options->server_port = 8060;
+	}
+	if(options->rtp_port_min == 0) {
+		options->rtp_port_min = 4000;
+	}
+	if(options->rtp_port_max == 0) {
+		options->rtp_port_max = 6000;
+	}
+
+	return APR_SUCCESS;
+}
+
+/** 
+ * Called back by openmcp client thread every time its ready for more audio 
+ * Reads data that was put into a shared fifo queue upon receiving audio frames
+ * from asr_feed()
+ */
+static apr_status_t openmrcp_recognizer_read_frame(audio_source_t *source, media_frame_t *frame)
+{
+	openmrcp_session_t *openmrcp_session = source->object;
+	frame->type = MEDIA_FRAME_TYPE_NONE;
+	apr_status_t result;
+	media_frame_t *queue_frame = NULL;
+
+
+	/* pop next media frame data from incoming queue into frame */
+	if(openmrcp_session->audio_input_queue) {
+		if (switch_queue_trypop(openmrcp_session->audio_input_queue, &queue_frame)) {
+			// switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "could not pop from queue\n");
+			result = MRCP_STATUS_FAILURE;
+		}
+		else {
+			frame->codec_frame.size = queue_frame->codec_frame.size; 
+			frame->codec_frame.buffer = queue_frame->codec_frame.buffer;
+			frame->type = MEDIA_FRAME_TYPE_AUDIO;
+			result = APR_SUCCESS;
+		}
+	}
+	else {
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "no audio_in queue\n");
+		result = MRCP_STATUS_FAILURE;
+	}
+
+	return result;
+}
+
+/** Read in the grammar and construct an MRCP Recognize message that has
+    The grammar attached as the payload */
+static mrcp_status_t openmrcp_recog_start(mrcp_client_context_t *context, openmrcp_session_t *openmrcp_session, char *path)
+{
+
+	mrcp_generic_header_t *generic_header;
+	apr_status_t rv;
+	apr_file_t *fp;
+	apr_pool_t *mp;
+	apr_finfo_t finfo;
+	char *buf1;
+	int bytes2read = 0;
+	
+	mrcp_message_t *mrcp_message = mrcp_client_context_message_get(context, openmrcp_session->client_session, openmrcp_session->channel, RECOGNIZER_RECOGNIZE);
+
+	if(!mrcp_message) {
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not create mrcp msg\n");
+		return MRCP_STATUS_FAILURE;
+	}
+
+	/* open the file with the grammar and read into char* buffer */
+	mp = mrcp_message->pool;
+	if ((rv = apr_file_open(&fp, path, APR_READ, APR_OS_DEFAULT, mp)) != APR_SUCCESS) {
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not read grammar\n");
+	    return -1;
+	}
+	rv = apr_file_info_get(&finfo, APR_FINFO_NORM, fp);
+	
+	buf1 = apr_palloc(mp, finfo.size);
+	bytes2read = finfo.size;
+	rv = apr_file_read(fp, buf1, &bytes2read);
+	generic_header = mrcp_generic_header_prepare(mrcp_message);
+	if(!generic_header) {
+	    return MRCP_STATUS_FAILURE;
+	}
+
+	generic_header->content_type = mrcp_str_pdup(mrcp_message->pool,"application/srgs+xml");
+	mrcp_generic_header_property_add(mrcp_message,GENERIC_HEADER_CONTENT_TYPE);
+	mrcp_message->body = mrcp_str_pdup(mrcp_message->pool,buf1);
+
+	/* send the MRCP RECOGNIZE message to MRCP server */
+	return mrcp_client_context_channel_modify(context, openmrcp_session->client_session, mrcp_message);
+
+}
+
+
+/**
+ * Freeswitch calls this from switch_ivr_detect_speech() and then adds a media
+ * bug to tap into the channel's audio, which will result in all data getting
+ * passed to asr_feed() and calls to asr_check_results() on each recevied frame.
+ * 
+ * This code expects certain one-time initialization of the openmrcp client
+ * engine/systeme to have already taken place.function to open the asr interface 
+ */
+static switch_status_t mrcprec_asr_open(switch_asr_handle_t *ah, char *codec, int rate, char *dest, switch_asr_flag_t *flags) 
+{
+	mrcp_client_t *mrcp_client = mrcprec_module.mrcp_client;
+	mrcp_client_context_t *mrcp_client_context = mrcprec_module.mrcp_client_context ;
+	openmrcp_client_options_t *options;
+	openmrcp_session_t *openmrcp_session;
+	audio_source_t *source;
+	
+	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "asr_open called, codec: %s, rate: %d\n", codec, rate);
+
+	/*! 
+	  sanity checking regarding codec/rate - NOTE: some of the other codecs and rates might
+	  actually work, but since they have not been tested yet, they are currently not
+	  allowed.  forcing MRCP to use 20 as the CODEC_FRAME_TIME_BASE effectively ensures
+	  that it matches with 16-bit audio at 8kz with 320 byte frames.  in testing, leaving
+	  CODEC_FRAME_TIME_BASE at 10 and using pop (instead of trypop) in 
+	  openmrcp_recognizer_read_frame() actually produces clean audio, however it causes 
+      other problems as the full channel/session cleanup never completes in openmrcp, most 
+	  likely due to a thread being blocked on a pop call from audio queue.  but with trypop 
+	  (to avoid the cleanup problem), it only produces clean audio when CODEC_FRAME_TIME_BASE 
+	  is set to 20.
+	 */
+	if (strcmp(codec,"L16")) {
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Sorry, only L16 codec supported\n");
+		return SWITCH_STATUS_GENERR;		
+	}
+	if (rate != 8000) {
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Sorry, only 8kz supported\n");
+		return SWITCH_STATUS_GENERR;		
+	}
+	if (CODEC_FRAME_TIME_BASE != 20) {
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "You must recompile openmrcp with #define CODEC_FRAME_TIME_BASE 20\n");
+		return SWITCH_STATUS_GENERR;				
+	}
+	else {
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "CODEC_FRAME_TIME_BASE: %d\n", CODEC_FRAME_TIME_BASE);
+	}
+
+	/* create session */
+	openmrcp_session = openmrcp_session_create();
+	if (!openmrcp_session) {
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "openmrcp_session creation FAILED\n");
+		return SWITCH_STATUS_GENERR;
+	}
+	openmrcp_session->client_session = mrcp_client_context_session_create(mrcp_client_context,openmrcp_session);
+	if (!openmrcp_session->client_session) {
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "openmrcp_session creation FAILED\n");
+		return SWITCH_STATUS_GENERR;
+	}
+
+	/* create audio source */
+	source = mrcp_palloc(openmrcp_session->pool,sizeof(audio_source_t)); 
+	source->method_set = &audio_source_method_set;
+	source->object = openmrcp_session;
+	openmrcp_session->source = source;
+	
+	/**
+	 * create a new fifo queue.  incoming audio received from freeswitch
+	 * will be put into this queue, and it will be later pulled out by 
+	 * the openmrcp client thread.
+	 */
+	if (switch_queue_create(&openmrcp_session->audio_input_queue, 10000, openmrcp_session->pool)) {
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "audio queue creation failed\n");
+		return SWITCH_STATUS_MEMERR;
+	}
+	
+	openmrcp_session->flags = *flags;
+	switch_mutex_init(&openmrcp_session->flag_mutex, SWITCH_MUTEX_NESTED, openmrcp_session->pool);
+
+	ah->private_info = openmrcp_session;
+
+	return SWITCH_STATUS_SUCCESS;
+
+}
+
+/* function to load a grammar to the asr interface */
+static switch_status_t mrcprec_asr_load_grammar(switch_asr_handle_t *ah, char *grammar, char *path)
+{
+	/** Read grammar from path and create and send and MRCP RECOGNIZE msg
+	    that has the grammar attached to body.   
+	
+	    TODO: - how does DEFINE-GRAMMAR fit into the picture here?  (if at all) 
+	*/
+	
+	openmrcp_session_t *openmrcp_session = (openmrcp_session_t *) ah->private_info;
+	mrcp_client_t *mrcp_client = mrcprec_module.mrcp_client;
+	mrcp_client_context_t *mrcp_client_context = mrcprec_module.mrcp_client_context;
+	audio_source_t *source = openmrcp_session->source;
+		
+	/* create recognizer channel, also starts outgoing rtp media */
+	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "Loading grammar\n");
+
+	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Create Recognizer Channel\n");
+	openmrcp_session->channel = mrcp_client_recognizer_channel_create(mrcp_client_context, openmrcp_session->client_session, source);
+
+	if (!openmrcp_session->channel) {
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Failed to create recognizer channel\n");
+		return SWITCH_STATUS_FALSE;
+	}
+	
+	/* wait for recognizer channel creation */
+	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "WAITING FOR CHAN CREATE\n");
+	if(openmrcp_session_wait_for_event(openmrcp_session, OPENMRCP_EVENT_CHANNEL_CREATE, OPENMRCP_WAIT_TIMEOUT) == MRCP_STATUS_SUCCESS) {
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Got channel creation event\n");
+		
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "WAITING FOR NONE\n");
+		if (openmrcp_session_wait_for_event(openmrcp_session,OPENMRCP_EVENT_NONE,200) == MRCP_STATUS_SUCCESS) {
+			switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "GOT NONE EVENT\n");
+		}
+		else {
+			switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "NEVER GOT NONE EVENT\n");
+		}
+		
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Start Recognizer\n");
+		openmrcp_recog_start(mrcp_client_context, openmrcp_session, path);
+		
+	}
+	else {
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "never got channel creation event\n");
+	}
+
+	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "Finished loading grammar\n");
+	return SWITCH_STATUS_SUCCESS;
+}
+
+
+/*! function to feed audio to the ASR*/
+static switch_status_t mrcprec_asr_feed(switch_asr_handle_t *ah, void *data, unsigned int len, switch_asr_flag_t *flags)
+{
+
+	openmrcp_session_t *openmrcp_session = (openmrcp_session_t *) ah->private_info;
+	mrcp_client_t *mrcp_client = mrcprec_module.mrcp_client;
+	mrcp_client_context_t *mrcp_client_context = mrcprec_module.mrcp_client_context;
+	media_frame_t *media_frame;
+	switch_byte_t *buffer;
+	
+	/* create new media frame */
+	media_frame = (media_frame_t *) switch_core_alloc(openmrcp_session->pool, sizeof(media_frame_t));
+	if (!media_frame) {
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "media_frame creation failed\n");
+		return SWITCH_STATUS_MEMERR;
+	}
+	
+	/** since *data buffer might get freed by caller (true or false?), allocate a
+		new buffer and copy *data into it **/
+	buffer = (switch_byte_t *) switch_core_alloc(openmrcp_session->pool, sizeof(switch_byte_t)*len);
+	if (!buffer) {
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not allocate buffer\n");
+		return SWITCH_STATUS_MEMERR;
+	}
+	buffer = memcpy(buffer, data, len);
+	
+	media_frame->codec_frame.buffer = buffer;
+	media_frame->codec_frame.size = len;  
+	media_frame->type = MEDIA_FRAME_TYPE_AUDIO;
+	
+	/* push audio to queue */
+	if (switch_queue_trypush(openmrcp_session->audio_input_queue, (void *) media_frame)) {
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "could not push audio to queue\n");
+		return SWITCH_STATUS_GENERR;
+	}
+
+	return SWITCH_STATUS_SUCCESS;
+}
+
+
+static switch_status_t mrcprec_asr_pause(switch_asr_handle_t *ah)
+{
+	openmrcp_session_t *openmrcp_session = (openmrcp_session_t *) ah->private_info;
+	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "mrcprec_asr_pause called\n");
+
+	return SWITCH_STATUS_SUCCESS;
+
+}
+
+static switch_status_t mrcprec_asr_resume(switch_asr_handle_t *ah)
+{
+	openmrcp_session_t *openmrcp_session = (openmrcp_session_t *) ah->private_info;
+	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "mrcprec_asr_resume called\n");
+
+	return SWITCH_STATUS_SUCCESS;
+}
+
+
+/*! function to unload a grammar to the asr interface */
+static switch_status_t mrcprec_asr_unload_grammar(switch_asr_handle_t *ah, char *grammar)
+{
+	openmrcp_session_t *openmrcp_session = (openmrcp_session_t *) ah->private_info;
+
+	return SWITCH_STATUS_SUCCESS;
+}
+
+/**
+ * Freeswitch calls this whenever the channel is hungup or the
+ * speech detection is stopped via a call to switch_ivr_stop_detect_speech()
+ */
+static switch_status_t mrcprec_asr_close(switch_asr_handle_t *ah, switch_asr_flag_t *flags)
+{
+	openmrcp_session_t *openmrcp_session = (openmrcp_session_t *) ah->private_info;
+
+	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "mrcprec_asr_close()\n");
+
+	mrcp_client_t *mrcp_client = mrcprec_module.mrcp_client;
+	mrcp_client_context_t *context = mrcprec_module.mrcp_client_context;
+
+	// TODO!! should we do a switch_pool_clear(switch_memory_pool_t *p) on the pool held
+	// by openmrcp_session?
+
+	// destroy channel
+	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Going to DESTROY CHANNEL\n");
+	mrcp_client_context_channel_destroy(context, openmrcp_session->client_session, 1);
+	if (openmrcp_session_wait_for_event(openmrcp_session,OPENMRCP_EVENT_CHANNEL_DESTROY,10000) == MRCP_STATUS_SUCCESS) {
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "OPENMRCP_EVENT_CHANNEL_DESTROY received\n");
+	}
+	else {
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "timed out waiting for OPENMRCP_EVENT_CHANNEL_DESTROY\n");
+	}
+
+	// terminate client session
+	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Going to TERMINATE SESSION\n");
+	mrcp_client_context_session_terminate(context, openmrcp_session->client_session);
+	if (openmrcp_session_wait_for_event(openmrcp_session,OPENMRCP_EVENT_SESSION_TERMINATE,10000) == MRCP_STATUS_SUCCESS) {
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "OPENMRCP_EVENT_SESSION_TERMINATE recevied\n");
+	}
+	else {
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "timed out waiting for OPENMRCP_EVENT_SESSION_TERMINATE\n");
+	}
+	
+	// destroy client session (NOTE: this sends a BYE to the other side)
+	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Going to DESTROY SESSION\n");
+	mrcp_client_context_session_destroy(context, openmrcp_session->client_session);
+
+	// destroys the openmrcp_session struct
+	openmrcp_session_destroy(openmrcp_session);
+
+	switch_set_flag(ah, SWITCH_ASR_FLAG_CLOSED);
+
+	return SWITCH_STATUS_SUCCESS;
+}
+
+
+/**
+ * Freeswitch calls this method from the speech_thread() thread body method
+ * in switch_ivr_async.c every time a new frame is received by the media bug
+ * attached to the audio channel.  If this method returns SWITCH_STATUS_SUCCESS,
+ * then Freeswitch will call mrcprec_asr_get_results() to get the result value.
+ */
+static switch_status_t mrcprec_asr_check_results(switch_asr_handle_t *ah, switch_asr_flag_t *flags)
+{
+	openmrcp_session_t *openmrcp_session = (openmrcp_session_t *) ah->private_info;
+	
+	switch_status_t rv = (switch_test_flag(openmrcp_session, FLAG_HAS_TEXT) || switch_test_flag(openmrcp_session, FLAG_BARGE)) ? SWITCH_STATUS_SUCCESS : SWITCH_STATUS_FALSE;
+	
+	return rv;
+}
+
+/*! This will be called after asr_check_results returns SUCCESS */
+static switch_status_t mrcprec_asr_get_results(switch_asr_handle_t *ah, char **xmlstr, switch_asr_flag_t *flags)
+{
+	openmrcp_session_t *openmrcp_session = (openmrcp_session_t *) ah->private_info;
+	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "mrcprec_asr_get_results called\n");
+	switch_status_t ret = SWITCH_STATUS_SUCCESS;
+
+	if (switch_test_flag(openmrcp_session, FLAG_BARGE)) {
+		switch_clear_flag_locked(openmrcp_session, FLAG_BARGE);
+		ret = SWITCH_STATUS_BREAK;
+	}
+	
+	if (switch_test_flag(openmrcp_session, FLAG_HAS_TEXT)) {
+		/*! 
+		   we have to extract the XML but stripping off the <?xml version="1.0"?>
+		   header.  the body looks like:
+		
+		   Completion-Cause:001 no-match
+		   Content-Type: application/nlsml+xml
+		   Content-Length: 260
+		 
+		  <?xml version="1.0"?>
+          <result xmlns="http://www.ietf.org/xml/ns/mrcpv2" xmlns:ex="http://www.example.com/example" score="100" grammar="session:request1 at form-level.store">
+            <interpretation>
+             <input mode="speech">open a</input>
+            </interpretation>
+          </result>
+		*/
+
+		char *marker = "?>";  // FIXME -- lame and brittle way of doing this.  use regex or better.
+		char *position = strstr(openmrcp_session->mrcp_message_last_rcvd->body, marker);
+		if (!position) {
+			switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Bad result received from mrcp server: %s", openmrcp_session->mrcp_message_last_rcvd->body);
+			ret = SWITCH_STATUS_FALSE;
+		}
+		else {
+			position += strlen(marker);
+			*xmlstr = strdup(position);
+		}
+
+		// since we are returning our result here, future calls to check_results
+		// should return False
+		switch_clear_flag_locked(openmrcp_session, FLAG_HAS_TEXT);
+		ret = SWITCH_STATUS_SUCCESS;	
+	}
+	return ret;
+}
+
+static switch_asr_interface_t mrcprec_asr_interface = {
+	/*.interface_name*/			"mrcprec",
+	/*.asr_open*/				mrcprec_asr_open,
+	/*.asr_load_grammar*/		mrcprec_asr_load_grammar,
+	/*.asr_unload_grammar*/		mrcprec_asr_unload_grammar,
+	/*.asr_close*/				mrcprec_asr_close,
+	/*.asr_feed*/				mrcprec_asr_feed,
+	/*.asr_resume*/				mrcprec_asr_resume,
+	/*.asr_pause*/				mrcprec_asr_pause,
+	/*.asr_check_results*/		mrcprec_asr_check_results,
+	/*.asr_get_results*/		mrcprec_asr_get_results
+};
+
+static switch_loadable_module_interface_t mrcprec_module_interface = {
+	/*.module_name */ modname,
+	/*.endpoint_interface */ NULL,
+	/*.timer_interface */ NULL,
+	/*.dialplan_interface */ NULL,
+	/*.codec_interface */ NULL,
+	/*.application_interface */ NULL,
+	/*.api_interface */ NULL,
+	/*.file_interface */ NULL,
+	/*.speech_interface */ NULL,
+	/*.directory_interface */ NULL,
+	/*.chat_interface */ NULL,
+	/*.say_interface */ NULL,
+	/*.asr_interface */ &mrcprec_asr_interface
+};
+
+
+static switch_status_t mrcp_init()
+{
+	/*!
+	Perform one-time initialization of client library
+	*/
+	
+	mrcp_mem_pool_t *pool;
+	mrcp_client_event_handler_t *client_event_handler;
+	mrcp_client_t *mrcp_client;
+	mrcp_client_context_t *mrcp_client_context;
+	openmrcp_client_options_t *options;
+	
+	pool = mrcp_global_pool_get();
+	options = mrcp_palloc(pool,sizeof(openmrcp_client_options_t));
+	client_event_handler = mrcp_palloc(pool,sizeof(mrcp_client_event_handler_t));
+	set_default_options(options);
+	
+	client_event_handler->on_session_initiate = openmrcp_on_session_initiate;
+	client_event_handler->on_session_terminate = openmrcp_on_session_terminate;
+	client_event_handler->on_channel_add = openmrcp_on_channel_add;
+	client_event_handler->on_channel_remove = openmrcp_on_channel_remove;
+	client_event_handler->on_channel_modify = openmrcp_on_channel_modify;
+
+	// create client context, which to must be passed to client engine 
+	mrcp_client_context = mrcp_client_context_create(&mrcprec_module,client_event_handler);
+	if(!mrcp_client_context) {
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "mrcp_client_context creation failed\n");
+		return SWITCH_STATUS_GENERR;
+	}
+	mrcprec_module.mrcp_client_context = mrcp_client_context;
+
+	// this basically starts a thread that pulls events from the event queue
+	// and handles them 
+	mrcp_client = openmrcp_client_start(options,mrcp_client_context);
+	if(!mrcp_client) {
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "openmrcp_client_start FAILED\n");
+		mrcp_client_context_destroy(mrcp_client_context);
+		return SWITCH_STATUS_GENERR;
+	}
+
+	mrcprec_module.mrcp_client = mrcp_client;
+
+	return SWITCH_STATUS_SUCCESS;
+}
+
+SWITCH_MODULE_LOAD_FUNCTION(mod_mrcprec_load)
+{
+
+	/* connect my internal structure to the blank pointer passed to me */
+	*module_interface = &mrcprec_module_interface;
+	
+	/* initialize openmrcp */
+	mrcp_global_init();
+	mrcp_init();
+
+	/* indicate that the module should continue to be loaded */
+	return SWITCH_STATUS_SUCCESS;
+}
+
+SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_mrcprec_shutdown)
+{
+	return SWITCH_STATUS_UNLOAD;
+}



More information about the Freeswitch-svn mailing list