[Freeswitch-branches] [commit] r5591 - freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp

Freeswitch SVN greenlizard at freeswitch.org
Mon Aug 13 14:10:16 EDT 2007


Author: greenlizard
Date: Mon Aug 13 14:10:16 2007
New Revision: 5591

Modified:
   freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/mod_openmrcp.c

Log:
going to merge mod_openmrcp to trunk

Modified: freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/mod_openmrcp.c
==============================================================================
--- freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/mod_openmrcp.c	(original)
+++ freeswitch/branches/greenlizard/src/mod/asr_tts/mod_openmrcp/mod_openmrcp.c	Mon Aug 13 14:10:16 2007
@@ -32,8 +32,29 @@
  *
  * Uses OpenMrcp (http://wiki.freeswitch.org/wiki/OpenMRCP) as the 
  * the client library.  
+ *
+ * TODO
+ * =======
+ *
+ * - MAJOR DESIGN ISSUE!!  It is way too expensive to be calling malloc on every audio frame,
+ * this needs to send the packet directly.  OpenMrcp will need a way to disable their own
+ * timer so that the fs timer is the only one driving the media.
+ * 
+ * - There are two memory pools in use.  One in asr_session which is managed
+ * by this module, and one in the switch_asr_handle_t, which is managed by freeswitch.
+ * These need to be consolidated into one.  (basically throw away the one in asr_session)
  * 
+ * - fs status codes (eg, SWITCH_STATUS_GENERR) and mrcp status codes (MRCP_STATUS_FAILURE)
+ * are intermixed badly.  this needs cleanup
+ * 
+ * - openmrcp_flush_tts, openmrcp_text_param_tts, openmrcp_numeric_param_tts, 
+ * openmrcp_float_param_tts need to have functionality added
+ *
+ * - fix audio problem with TTS, convert from using queue to using a switch_buffer
+ * (in progress)
+ *
  */
+
 #ifdef __ICC
 #pragma warning (disable:188)
 #endif
@@ -63,6 +84,8 @@
 #include <switch.h>
 	
 #define OPENMRCP_WAIT_TIMEOUT 5000
+#define MY_BUF_LEN 1024 * 32
+#define MY_BLOCK_SIZE MY_BUF_LEN
 
 SWITCH_MODULE_LOAD_FUNCTION(mod_openmrcp_load);
 SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_openmrcp_shutdown);
@@ -120,16 +143,16 @@
 struct tts_session_t {
 	mrcp_session_t        *client_session;
 	mrcp_client_channel_t *channel;
-	switch_queue_t        *audio_queue;
+	switch_queue_t        *audio_queue;  // TO BE REMOVED
 	switch_queue_t        *event_queue;
-	audio_sink_t        *sink;
+	switch_mutex_t        *audio_lock;
+	switch_buffer_t       *audio_buffer;
+	audio_sink_t          *sink;
 	apr_pool_t            *pool;
 	switch_speech_flag_t  flags;
 	switch_mutex_t        *flag_mutex;
 };
 
-long add_up_buffer(switch_byte_t *buffer, size_t len); // TEMP DEBUG
-
 static apr_status_t openmrcp_recognizer_read_frame(audio_source_t *source, media_frame_t *frame);
 static apr_status_t openmrcp_tts_write_frame(audio_sink_t *sink, media_frame_t *frame);
 
@@ -522,9 +545,6 @@
 
 	len = frame->codec_frame.size;
 
-	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "add_up frame: %li\n", 
-					  add_up_buffer(frame->codec_frame.buffer, len));
-
 	/* create new media frame */
 	media_frame = (media_frame_t *) switch_core_alloc(tts_session->pool, sizeof(media_frame_t));
 	if (!media_frame) {
@@ -547,10 +567,6 @@
 	media_frame->type = MEDIA_FRAME_TYPE_AUDIO;
 
 
-	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "add_up media_frame: %li\n", 
-					  add_up_buffer(media_frame->codec_frame.buffer, len));
-
-
 	/* push audio to queue */
 	if (switch_queue_trypush(tts_session->audio_queue, (void *) media_frame)) {
 		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "could not push audio to queue\n");
@@ -561,15 +577,63 @@
 
 }
 
-long add_up_buffer(switch_byte_t *buffer, size_t len) {
-	// add up contents in buffer
-	long result = 0;
-	for (int i=0; i<len; i++) {
-		result =+ buffer[i];
+
+/**
+ * Called back by openmrcp client thread every time it receives audio 
+ * from the TTS server we are connected to.  Puts audio in a queueu
+ * and it will be pulled out from read_tts
+ */
+static apr_status_t openmrcp_tts_write_frame_NEW(audio_sink_t *sink, media_frame_t *frame)
+{
+	tts_session_t *tts_session = sink->object;	
+	media_frame_t *media_frame;
+	switch_byte_t *buffer;
+	size_t len;
+	apr_status_t status = 
+
+	len = frame->codec_frame.size;
+
+	/* create new media frame */
+	media_frame = (media_frame_t *) switch_core_alloc(tts_session->pool, sizeof(media_frame_t));
+	if (!media_frame) {
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "media_frame creation failed\n");
+		return SWITCH_STATUS_MEMERR;
 	}
-	return result;
+	
+	/**
+	 * since *frame might get freed by caller (true or false?), allocate a
+	 * new buffer and copy *data into it.
+	 **/
+	buffer = (switch_byte_t *) switch_core_alloc(tts_session->pool, sizeof(switch_byte_t)*len);
+	if (!buffer) {
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not allocate buffer\n");
+		return SWITCH_STATUS_MEMERR;
+	}
+	buffer = memcpy(buffer, frame->codec_frame.buffer, len);
+	media_frame->codec_frame.buffer = buffer;
+	media_frame->codec_frame.size = len;  
+	media_frame->type = MEDIA_FRAME_TYPE_AUDIO;
+
+
+	/* add audio to buffer */
+	switch_mutex_lock(tts_session->audio_lock);
+	if (switch_buffer_write(tts_session->audio_buffer, buffer, len) == 0) {
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "Could not write to buffer\n");
+	}
+	switch_mutex_unlock(tts_session->audio_lock);
+
+	switch_mutex_unlock(tts_session->audio_lock);
+
+	if (switch_queue_trypush(tts_session->audio_queue, (void *) media_frame)) {
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "could not push audio to queue\n");
+		return MRCP_STATUS_FAILURE;
+	}
+
+	return MRCP_STATUS_SUCCESS;
+
 }
 
+
 /** 
  * Called back by openmcp client thread every time its ready for more audio to send
  * the recognition server we are connected to.  Reads data that was put into a 
@@ -1047,6 +1111,15 @@
 		return SWITCH_STATUS_MEMERR;
 	}
 
+	/* create mutex that will be used to lock audio buffer */
+	switch_mutex_init(&tts_session->audio_lock, SWITCH_MUTEX_NESTED, sh->memory_pool);
+
+	/* create audio buffer */
+	if (switch_buffer_create_dynamic(&tts_session->audio_buffer, MY_BLOCK_SIZE, MY_BUF_LEN, 0) != SWITCH_STATUS_SUCCESS) {
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_CRIT, "Write Buffer Creation Failed!\n");
+		return SWITCH_STATUS_MEMERR;
+	}
+
 	
 	tts_session->flags = *flags;
 	switch_mutex_init(&tts_session->flag_mutex, SWITCH_MUTEX_NESTED, tts_session->pool);
@@ -1093,11 +1166,10 @@
 	/* wait for synthesizer channel creation */
 	if(wait_for_event(tts_session->event_queue,OPENMRCP_EVENT_CHANNEL_CREATE,5000) == MRCP_STATUS_SUCCESS) {
 		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "Got channel create event\n");
-		wait_for_event(tts_session->event_queue,OPENMRCP_EVENT_NONE,1000);
+		wait_for_event(tts_session->event_queue,OPENMRCP_EVENT_NONE,1000);  // XXX: what are we waiting for??
 		/* speak */
 		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "Going to speak\n");
 		synth_speak(tts_client_context,tts_session, text); 
-		// wait_for_event(tts_session->event_queue,OPENMRCP_EVENT_CHANNEL_MODIFY,5000);
 
 	}
 	else {
@@ -1105,20 +1177,14 @@
 		return SWITCH_STATUS_FALSE;	
 	}
 
-
-
 	return SWITCH_STATUS_SUCCESS;	
 }
 
-static void openmrcp_flush_tts(switch_speech_handle_t *sh)
-{
-	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "flush_tts called\n");
-}
-
 /**
  * Freeswitch calls this when its ready to read datalen bytes of data.
  * 
- * TODO: check the blocking flag and act accordingly 
+ * TODO: check the blocking flag passed in flags and act accordingly  
+ *       (see mod_cepstral.c)
  */
 static switch_status_t openmrcp_read_tts(switch_speech_handle_t *sh, void *data, size_t *datalen, uint32_t *rate, switch_speech_flag_t *flags)
 {
@@ -1132,7 +1198,6 @@
 		return SWITCH_STATUS_SUCCESS;	
 	}
 	else {
-
 		memcpy(data, queue_frame->codec_frame.buffer, queue_frame->codec_frame.size);
 		*datalen = queue_frame->codec_frame.size;
 		*rate = 8000;
@@ -1141,6 +1206,12 @@
 
 }
 
+
+static void openmrcp_flush_tts(switch_speech_handle_t *sh)
+{
+	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "flush_tts called\n");
+}
+
 static void openmrcp_text_param_tts(switch_speech_handle_t *sh, char *param, char *val)
 {
 



More information about the Freeswitch-branches mailing list