From ad9a0e84888027b08b39c9ccb4b1b18a2b2ea064 Mon Sep 17 00:00:00 2001 From: Autobuilder at buildslave-ubuntu-12-32 Date: Thu, 7 Nov 2013 09:34:28 +0100 Subject: refcounts, many connected client, audio bugfix --- .../plugins/invoker/miles/MilesSessionInvoker.cpp | 94 ++++++++++++++++------ .../plugins/invoker/miles/MilesSessionInvoker.h | 2 + 2 files changed, 71 insertions(+), 25 deletions(-) diff --git a/src/uscxml/plugins/invoker/miles/MilesSessionInvoker.cpp b/src/uscxml/plugins/invoker/miles/MilesSessionInvoker.cpp index b45a324..8bc352b 100644 --- a/src/uscxml/plugins/invoker/miles/MilesSessionInvoker.cpp +++ b/src/uscxml/plugins/invoker/miles/MilesSessionInvoker.cpp @@ -44,10 +44,11 @@ MilesSessionInvoker::MilesSessionInvoker() { /* Initalize Miles */ miles_init(); - /* media buffers */ + /* set up media buffers */ init_media_buffers(); _isRunning = false; + num_connected = 0; } MilesSessionInvoker::~MilesSessionInvoker() { @@ -235,9 +236,9 @@ void MilesSessionInvoker::processEventStart(const std::string& origin, const std Event ev; ev.data.compound["origin"] = origin; //std::cout << req; - if(_isRunning) { - LOG(ERROR) << "already connected"; - ev.name = "start.error"; + if(num_connected>0) { + num_connected++; + ev.name = "start.reply"; returnEvent(ev); return; } @@ -254,6 +255,7 @@ void MilesSessionInvoker::processEventStart(const std::string& origin, const std } LOG(ERROR) << "session set up"; + /* Set up audio and video RTP sockets */ video_rtp_in_socket = miles_net_setup_udp_socket((char*)reflector.c_str(), video_port, video_port, 10, 16000); audio_rtp_in_socket = miles_net_setup_udp_socket((char*)reflector.c_str(), audio_port, audio_port, 10, 16000); @@ -311,6 +313,7 @@ void MilesSessionInvoker::processEventStart(const std::string& origin, const std memset(confero_text_msg_buf, 0, 1000); _isRunning = true; + num_connected=1; _reflector = reflector; _userId = userid; _session = session; @@ -326,12 +329,18 @@ void MilesSessionInvoker::processEventStop(const std::string& origin) { Event ev; ev.data.compound["origin"] = origin; - if(!_isRunning) { + if(num_connected==0) { LOG(ERROR) << "not connected"; ev.name = "stop.error"; returnEvent(ev); return; } + num_connected--; + if(num_connected>0) { + ev.name = "stop.reply"; + returnEvent(ev); + return; + } int rv = miles_disconnect_reflector_session((char*)_reflector.c_str(), (char*)_session.c_str()); if (!rv) { LOG(ERROR) << "Could not disconnect from reflector session"; @@ -339,7 +348,11 @@ void MilesSessionInvoker::processEventStop(const std::string& origin) { returnEvent(ev); return; } - free_media_buffers(); + /* Unregister RTCP APP handler for text messages */ + rv = miles_rtp_unregister_rtcp_app_handler("text"); + if(rv==0) { + LOG(ERROR) << "Error registering text message callback"; + } _isRunning = false; ev.name = "stop.reply"; returnEvent(ev); @@ -350,7 +363,7 @@ void MilesSessionInvoker::processEventParticipants(const std::string& origin) { Event ev; ev.data.compound["origin"] = origin; - if(!_isRunning) { + if(num_connected==0) { LOG(ERROR) << "not connected"; ev.name = "participants.error"; returnEvent(ev); @@ -371,7 +384,7 @@ void MilesSessionInvoker::processEventParticipants(const std::string& origin) { void MilesSessionInvoker::processEventThumbnail(const std::string& origin, const std::string& userid) { Event ev; ev.data.compound["origin"] = origin; - if(!_isRunning) { + if(num_connected==0) { LOG(ERROR) << "not connected"; ev.name = "thumbnail.error"; returnEvent(ev); @@ -475,8 +488,14 @@ void MilesSessionInvoker::processEventPostText(const std::string& origin, const int n, length; Event ev; - ev.name = "posttext.reply"; ev.data.compound["origin"] = origin; + if(num_connected==0) { + LOG(ERROR) << "not connected"; + ev.name = "posttext.error"; + returnEvent(ev); + return; + } + ev.name = "posttext.reply"; returnEvent(ev); if(out_rtcp_video_stream==NULL) return; @@ -492,7 +511,7 @@ void MilesSessionInvoker::processEventPostText(const std::string& origin, const void MilesSessionInvoker::processEventGetText(const std::string& origin) { Event ev; ev.data.compound["origin"] = origin; - if(!_isRunning) { + if(num_connected==0) { LOG(ERROR) << "not connected"; ev.name = "gettext.error"; returnEvent(ev); @@ -500,12 +519,6 @@ void MilesSessionInvoker::processEventGetText(const std::string& origin) { } ev.name = "gettext.reply"; -#if 0 - if (rand() % 5 == 0) { // return some mocked up chat message - ev.data.compound["message"] = Data(".. and then she was all like: aren't we supposed to discuss work related stuff?", Data::VERBATIM); - ev.data.compound["user"] = Data("username1", Data::VERBATIM); - } -#else if(confero_text_msg_available) { strcpy(text_msg_buf, confero_text_msg_buf); ev.data.compound["message"] = Data(text_msg_buf, Data::VERBATIM); @@ -514,7 +527,6 @@ void MilesSessionInvoker::processEventGetText(const std::string& origin) { memset(confero_text_msg_buf, 0, 1000); confero_text_msg_available = 0; } -#endif returnEvent(ev); } @@ -533,6 +545,19 @@ void MilesSessionInvoker::processVideo() { if(video_grabber_available && sendvideo_enabled) video_transmitter(video_grabber, video_encoder, out_rtp_video_stream, out_rtcp_video_stream); } + /* done, clean up */ + if(video_grabber_available) { + miles_rtp_destroy_out_stream(out_rtp_video_stream); + miles_video_grabber_destroy(video_grabber); + miles_video_codec_destroy_encoder(video_encoder); + video_grabber_available = 0; + sendvideo_enabled = 0; + } + miles_rtp_destroy_session(video_session); + miles_list_destroy(thumb_list); + thumb_list = NULL; + miles_net_socket_close(video_rtp_in_socket); + miles_net_socket_close(video_rtcp_in_socket); } void MilesSessionInvoker::processAudio() { @@ -541,9 +566,29 @@ void MilesSessionInvoker::processAudio() { if(audio_available && sendaudio_enabled) audio_transmitter(audio_dev, audio_encoder, out_rtp_audio_stream, out_rtcp_audio_stream); } + /* done, clean up */ + if(audio_available) { + miles_rtp_destroy_out_stream(out_rtp_audio_stream); + if(audio_dev_playback) + miles_audio_device_close(MILES_AUDIO_IO_OPENAL, audio_dev_playback, 0); + if(audio_dev) + miles_audio_device_close(MILES_AUDIO_IO_OPENAL, audio_dev_playback, 1); + miles_audio_codec_destroy_encoder(audio_encoder); + audio_available = 0; + sendaudio_enabled = 0; + } + miles_rtp_destroy_session(audio_session); + miles_net_socket_close(audio_rtp_in_socket); + miles_net_socket_close(audio_rtcp_in_socket); } int MilesSessionInvoker::setup_audio() { + /* Check that we have OpeanAL audio */ + if(!miles_audio_io_is_supported(MILES_AUDIO_IO_OPENAL)) { + fprintf(stderr, "OpenAL audio i/o not supported on this platform.\n"); + return 0; + } + /* Initialize and configure audio encoder */ audio_encoder = miles_audio_codec_init_encoder(); audio_encoder->codec_id = miles_audio_codec_get_encoder_for_rtp_payload_type(MILES_RTP_PAYLOAD_TYPE_L16); @@ -557,10 +602,9 @@ int MilesSessionInvoker::setup_audio() { LOG(ERROR) << "Couldn't set up audio codec"; return 0; } - LOG(ERROR) << "audio enc set up"; /* Set up audio grabber */ - int n = miles_audio_device_get_supported_devices(MILES_AUDIO_DEVICE_OPENAL, &supported_audio_devices); + int n = miles_audio_device_get_supported_devices(MILES_AUDIO_IO_OPENAL, &supported_audio_devices); if(n<=0) { /* No audio device available */ LOG(ERROR) << "No audio device available"; @@ -568,7 +612,7 @@ int MilesSessionInvoker::setup_audio() { } /* Use first device that supports capture */ for(int i=0; ichunk_size != size || audio_dev_playback->sample_rate != sample_rate || audio_dev_playback->format != audio_format || audio_dev_playback->bytes_per_sample != bps) { if(audio_dev_playback) - miles_audio_device_close(MILES_AUDIO_DEVICE_OPENAL, audio_dev_playback, 0); - audio_dev_playback = miles_audio_device_open(MILES_AUDIO_DEVICE_OPENAL, audio_dev_playback_id, audio_format, sample_rate, bps, 1, size, 0); + miles_audio_device_close(MILES_AUDIO_IO_OPENAL, audio_dev_playback, 0); + audio_dev_playback = miles_audio_device_open(MILES_AUDIO_IO_OPENAL, audio_dev_playback_id, audio_format, sample_rate, bps, 1, size, 0); if(audio_dev_playback == NULL) return; } /* play audio */ - miles_audio_device_write(MILES_AUDIO_DEVICE_OPENAL, audio_dev_playback, buf, size); + miles_audio_device_write(MILES_AUDIO_IO_OPENAL, audio_dev_playback, buf, size); } /** @@ -945,7 +989,7 @@ int MilesSessionInvoker::audio_transmitter(struct miles_audio_device *dev, struc /* Send RTCP packets, if due */ miles_rtp_send_rtcp(out_rtcp_audio_stream); - n = miles_audio_device_read(MILES_AUDIO_DEVICE_OPENAL, dev, audio_read_buf, codec_ctx->chunk_size); + n = miles_audio_device_read(MILES_AUDIO_IO_OPENAL, dev, audio_read_buf, codec_ctx->chunk_size); if(n <= 0) return 0; if(dev->format != codec_ctx->input_format) { diff --git a/src/uscxml/plugins/invoker/miles/MilesSessionInvoker.h b/src/uscxml/plugins/invoker/miles/MilesSessionInvoker.h index eeb6e7c..2191b8a 100644 --- a/src/uscxml/plugins/invoker/miles/MilesSessionInvoker.h +++ b/src/uscxml/plugins/invoker/miles/MilesSessionInvoker.h @@ -27,6 +27,7 @@ extern "C" { #include "miles/network.h" #include "miles/rtp.h" #include "miles/audio_codec.h" +#include "miles/audio_io.h" #include "miles/audio_device.h" #include "miles/video_codec.h" #include "miles/video_grabber.h" @@ -150,6 +151,7 @@ protected: bool _isRunning; + int num_connected; std::string _userId, _reflector, _session; tthread::thread* _videoThread; tthread::thread* _audioThread; -- cgit v0.12 From fb91e9aa1cfa84f471024ac8de63b2ffebf934cb Mon Sep 17 00:00:00 2001 From: Autobuilder at buildslave-ubuntu-12-32 Date: Thu, 7 Nov 2013 09:53:48 +0100 Subject: better media buffer handling --- .../plugins/invoker/miles/MilesSessionInvoker.cpp | 51 ++++++++++++++++++++-- .../plugins/invoker/miles/MilesSessionInvoker.h | 4 ++ 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/src/uscxml/plugins/invoker/miles/MilesSessionInvoker.cpp b/src/uscxml/plugins/invoker/miles/MilesSessionInvoker.cpp index 8bc352b..dcefc0f 100644 --- a/src/uscxml/plugins/invoker/miles/MilesSessionInvoker.cpp +++ b/src/uscxml/plugins/invoker/miles/MilesSessionInvoker.cpp @@ -41,12 +41,9 @@ bool pluginConnect(pluma::Host& host) { #endif MilesSessionInvoker::MilesSessionInvoker() { - /* Initalize Miles */ + /* Initialize Miles */ miles_init(); - /* set up media buffers */ - init_media_buffers(); - _isRunning = false; num_connected = 0; } @@ -112,6 +109,45 @@ void MilesSessionInvoker::free_media_buffers() { text_msg_available = 0; } +void MilesSessionInvoker::free_video_buffers() { + if(video_out_buf) + free(video_out_buf); + video_out_buf = NULL; + if(encoded_out_img) + free(encoded_out_img); + encoded_out_img = NULL; + if(render_img) + free(render_img); + render_img = NULL; + render_img_size = 0; + if(video_data) + free(video_data); + video_data = NULL; +} + +void MilesSessionInvoker::free_audio_buffers() { + if(audio_in_buf) + free(audio_in_buf); + audio_in_buf = NULL; + if(audio_data) + free(audio_data); + audio_data = NULL; + video_data = NULL; + if(encoded_out_audio) + free(encoded_out_audio); + encoded_out_audio = NULL; + if(audio_read_buf) + free(audio_read_buf); + audio_read_buf = NULL; +} + +void MilesSessionInvoker::free_text_buffers() { + if(text_msg_buf) + free(text_msg_buf); + text_msg_buf = NULL; + text_msg_available = 0; +} + // Yes, sort of ugly... char confero_text_msg_buf[1000]; int confero_text_msg_available = 0; @@ -255,6 +291,8 @@ void MilesSessionInvoker::processEventStart(const std::string& origin, const std } LOG(ERROR) << "session set up"; + /* set up media buffers */ + init_media_buffers(); /* Set up audio and video RTP sockets */ video_rtp_in_socket = miles_net_setup_udp_socket((char*)reflector.c_str(), video_port, video_port, 10, 16000); @@ -354,6 +392,7 @@ void MilesSessionInvoker::processEventStop(const std::string& origin) { LOG(ERROR) << "Error registering text message callback"; } _isRunning = false; + free_text_buffers(); ev.name = "stop.reply"; returnEvent(ev); LOG(ERROR) << "disconnected from reflector session"; @@ -558,6 +597,8 @@ void MilesSessionInvoker::processVideo() { thumb_list = NULL; miles_net_socket_close(video_rtp_in_socket); miles_net_socket_close(video_rtcp_in_socket); + + free_video_buffers(); } void MilesSessionInvoker::processAudio() { @@ -580,6 +621,8 @@ void MilesSessionInvoker::processAudio() { miles_rtp_destroy_session(audio_session); miles_net_socket_close(audio_rtp_in_socket); miles_net_socket_close(audio_rtcp_in_socket); + + free_video_buffers(); } int MilesSessionInvoker::setup_audio() { diff --git a/src/uscxml/plugins/invoker/miles/MilesSessionInvoker.h b/src/uscxml/plugins/invoker/miles/MilesSessionInvoker.h index 2191b8a..552297c 100644 --- a/src/uscxml/plugins/invoker/miles/MilesSessionInvoker.h +++ b/src/uscxml/plugins/invoker/miles/MilesSessionInvoker.h @@ -140,6 +140,10 @@ protected: void init_media_buffers(); void free_media_buffers(); + void free_video_buffers(); + void free_audio_buffers(); + void free_text_buffers(); + void render_video_image(char *img, int width, int height, int img_format); void playback_audio(u_int32_t ssrc, char *buf, int sample_rate, int bps, int audio_format, int size); int video_receiver(struct miles_rtp_in_stream *rtp_stream, char *data, int bytes_read); -- cgit v0.12 From 1dca74be3a40cb878cd284d3572fca020173113a Mon Sep 17 00:00:00 2001 From: Autobuilder at buildslave-ubuntu-12-32 Date: Thu, 7 Nov 2013 11:20:59 +0100 Subject: multiple video streams --- apps/samples/miles/miles.html | 6 ++-- apps/samples/miles/miles.js | 1 + apps/samples/miles/miles.scxml | 6 ++-- .../plugins/invoker/miles/MilesSessionInvoker.cpp | 41 +++++++++++----------- .../plugins/invoker/miles/MilesSessionInvoker.h | 1 + 5 files changed, 29 insertions(+), 26 deletions(-) diff --git a/apps/samples/miles/miles.html b/apps/samples/miles/miles.html index 08cfeca..5028621 100644 --- a/apps/samples/miles/miles.html +++ b/apps/samples/miles/miles.html @@ -16,9 +16,9 @@ diff --git a/apps/samples/miles/miles.js b/apps/samples/miles/miles.js index 4f31d68..9419a8e 100644 --- a/apps/samples/miles/miles.js +++ b/apps/samples/miles/miles.js @@ -60,6 +60,7 @@ function Miles(element, params) { if (params && params.reflectorIp) reflectorIp = params.reflectorIp; if (params && params.email) email = params.email; if (params && params.problemName) problemName = params.problemName; + if (params && params.remoteEmail) remoteEmail = params.remoteEmail; // called when dojo loaded all requirements below this.connect = function() { diff --git a/apps/samples/miles/miles.scxml b/apps/samples/miles/miles.scxml index 22ebdda..154e4d6 100644 --- a/apps/samples/miles/miles.scxml +++ b/apps/samples/miles/miles.scxml @@ -49,7 +49,7 @@ - + @@ -58,7 +58,7 @@ - + @@ -72,7 +72,7 @@ - + diff --git a/src/uscxml/plugins/invoker/miles/MilesSessionInvoker.cpp b/src/uscxml/plugins/invoker/miles/MilesSessionInvoker.cpp index dcefc0f..b36935c 100644 --- a/src/uscxml/plugins/invoker/miles/MilesSessionInvoker.cpp +++ b/src/uscxml/plugins/invoker/miles/MilesSessionInvoker.cpp @@ -437,31 +437,31 @@ void MilesSessionInvoker::processEventThumbnail(const std::string& origin, const ev.name = "thumbnail.reply"; - int has_thumb = 0; + struct thumb_entry *use_thumb = NULL; struct miles_rtp_in_stream *rtps; struct miles_list *p; struct thumb_entry *te; _mutex.lock(); - if(video_session->instreams) { - rtps = video_session->instreams->stream; - if(rtps) { - p = thumb_list; - while(p) { - te = (struct thumb_entry *)p->item; - if(te->ssrc == rtps->ssrc) { - break; - } - p = p->next; - } - if(p) { - has_thumb = 1; - ev.data.compound["image"] = Data(te->img_buf, te->img_size, "image/jpeg"); - } + // Find thumbnail of user + p = thumb_list; + while(p) { + te = (struct thumb_entry *)p->item; + if(te->userid && strcmp(te->userid, userid.c_str()) == 0) { + use_thumb = te; + break; } - } - if(!has_thumb) { - // Return empty face image - ev.data.compound["image"] = Data(imageContent.data(), imageContent.size(), "image/jpeg"); + if(te->userid==NULL && use_thumb == NULL) { + use_thumb = te; + } + p = p->next; + } + if(!p && use_thumb) + use_thumb->userid = strdup(userid.c_str()); + if(use_thumb) { + ev.data.compound["image"] = Data(use_thumb->img_buf, use_thumb->img_size, "image/jpeg"); + } else { + // Return empty face image + ev.data.compound["image"] = Data(imageContent.data(), imageContent.size(), "image/jpeg"); } _mutex.unlock(); @@ -845,6 +845,7 @@ int MilesSessionInvoker::video_receiver(struct miles_rtp_in_stream *rtp_stream, p = miles_list_append(thumb_list, te); te->ssrc = rtp_stream->ssrc; te->window_ctx = NULL; + te->userid = NULL; te->img_buf = (char *)malloc(bytes_read); te->buf_size = bytes_read; te->img_size = 0; diff --git a/src/uscxml/plugins/invoker/miles/MilesSessionInvoker.h b/src/uscxml/plugins/invoker/miles/MilesSessionInvoker.h index 552297c..fec2243 100644 --- a/src/uscxml/plugins/invoker/miles/MilesSessionInvoker.h +++ b/src/uscxml/plugins/invoker/miles/MilesSessionInvoker.h @@ -51,6 +51,7 @@ struct thumb_entry { int img_format; // JPEG or PNG image char *decode_buf; u_int32_t ssrc; + char *userid; // The user id assigned to the video stream void *window_ctx; // The context of the window popped up when the thumbnail is clicked. }; -- cgit v0.12