/* * GPAC - Multimedia Framework C SDK * * Copyright (c) Jean Le Feuvre 2000-2005 * All rights reserved * * This file is part of GPAC / Scene Rendering sub-project * * GPAC is free software; you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License as published by * the Free Software Foundation; either version 2, or (at your option) * any later version. * * GPAC is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; see the file COPYING. If not, write to * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA. * */ #include /*max number of channels we support in mixer*/ #define GF_SR_MAX_CHANNELS 16 /* Notes about the mixer: 1- spatialization is out of scope for the mixer (eg that's the sound node responsability) 2- mixing is performed by resampling input source & deinterleaving its channels into dedicated buffer. We could directly deinterleave in the main mixer ouput buffer, but this would prevent any future gain correction. */ typedef struct { GF_AudioInterface *src; s32 *ch_buf[GF_SR_MAX_CHANNELS]; /*resampled buffer*/ u32 buffer_size; u32 bytes_per_sec; Bool has_prev; s32 last_channels[GF_SR_MAX_CHANNELS]; u32 in_bytes_used, out_samples_written, out_samples_to_write; Fixed speed; Fixed pan[6]; } MixerInput; struct __audiomix { /*src*/ GF_List *sources; /*output config*/ u32 sample_rate; u32 nb_channels; u32 bits_per_sample; u32 channel_cfg; GF_Mutex *mx; /*if set forces stereo/mono*/ Bool force_channel_out; /*set to true by mixer when detecting an audio config change*/ Bool must_reconfig; Bool isEmpty; /*set to non null if this outputs directly to the driver, in which case audio formats have to be checked*/ struct _audio_render *ar; s32 *output; u32 output_size; }; GF_AudioMixer *gf_mixer_new(struct _audio_render *ar) { GF_AudioMixer *am; am = (GF_AudioMixer *) malloc(sizeof(GF_AudioMixer)); if (!am) return NULL; memset(am, 0, sizeof(GF_AudioMixer)); am->mx = gf_mx_new(); am->sources = gf_list_new(); am->isEmpty = 1; am->ar = ar; am->sample_rate = 44100; am->bits_per_sample = 16; am->nb_channels = 2; am->output = NULL; am->output_size = 0; return am; } Bool gf_mixer_must_reconfig(GF_AudioMixer *am) { return am->must_reconfig; } void gf_mixer_del(GF_AudioMixer *am) { gf_list_del(am->sources); gf_mx_del(am->mx); if (am->output) free(am->output); free(am); } void gf_mixer_remove_all(GF_AudioMixer *am) { u32 j; gf_mixer_lock(am, 1); while (gf_list_count(am->sources)) { MixerInput *in = (MixerInput *)gf_list_get(am->sources, 0); gf_list_rem(am->sources, 0); for (j=0; jch_buf[j]) free(in->ch_buf[j]); } free(in); } am->isEmpty = 1, gf_mixer_lock(am, 0); } Bool gf_mixer_is_src_present(GF_AudioMixer *am, GF_AudioInterface *ifce) { MixerInput *in; u32 i = 0; while ((in = (MixerInput *)gf_list_enum(am->sources, &i))) { if (in->src == ifce) return 1; } return 0; } u32 gf_mixer_get_src_count(GF_AudioMixer *am) { return gf_list_count(am->sources); } void gf_mixer_force_chanel_out(GF_AudioMixer *am, u32 num_channels) { am->force_channel_out = 1; am->nb_channels = num_channels; } u32 gf_mixer_get_block_align(GF_AudioMixer *am) { return am->nb_channels*am->bits_per_sample/8; } void gf_mixer_lock(GF_AudioMixer *am, Bool lockIt) { //GF_LOG(GF_LOG_DEBUG, GF_LOG_CORE, ("[AudioRender] Thread ID %d is %s the audio mixer\n", gf_th_id(), lockIt ? "locking" : "unlocking" )); if (lockIt) { gf_mx_p(am->mx); } else { gf_mx_v(am->mx); } } void gf_mixer_add_input(GF_AudioMixer *am, GF_AudioInterface *src) { MixerInput *in; if (gf_mixer_is_src_present(am, src)) return; gf_mixer_lock(am, 1); GF_SAFEALLOC(in, MixerInput); in->src = src; gf_list_add(am->sources, in); am->must_reconfig = 1; am->isEmpty = 0; gf_mixer_lock(am, 0); } void gf_mixer_remove_input(GF_AudioMixer *am, GF_AudioInterface *src) { u32 i, j, count; if (am->isEmpty) return; gf_mixer_lock(am, 1); count = gf_list_count(am->sources); for (i=0; isources, i); if (in->src != src) continue; gf_list_rem(am->sources, i); for (j=0; jch_buf[j]) free(in->ch_buf[j]); } free(in); break; } am->isEmpty = gf_list_count(am->sources) ? 0 : 1; /*we don't ask for reconfig when removing a node*/ gf_mixer_lock(am, 0); } static GF_Err get_best_samplerate(GF_AudioMixer *am, u32 *out_sr, u32 *out_ch, u32 *out_bps) { if (!am->ar) return GF_OK; if (!am->ar->audio_out || !am->ar->audio_out->QueryOutputSampleRate) return GF_OK; return am->ar->audio_out->QueryOutputSampleRate(am->ar->audio_out, out_sr, out_ch, out_bps); } void gf_mixer_get_config(GF_AudioMixer *am, u32 *outSR, u32 *outCH, u32 *outBPS, u32 *outChCfg) { (*outBPS) = am->bits_per_sample; (*outCH) = am->nb_channels; (*outSR) = am->sample_rate; (*outChCfg) = am->channel_cfg; } void gf_mixer_set_config(GF_AudioMixer *am, u32 outSR, u32 outCH, u32 outBPS, u32 outChCfg) { if ((am->bits_per_sample == outBPS) && (am->nb_channels == outCH) && (am->sample_rate==outSR) && (am->channel_cfg == outChCfg)) return; gf_mixer_lock(am, 1); am->bits_per_sample = outBPS; if (!am->force_channel_out) am->nb_channels = outCH; if (get_best_samplerate(am, &outSR, &outCH, &outBPS) == GF_OK) { am->sample_rate = outSR; if (outCH>2) am->channel_cfg = outChCfg; else if (outCH==2) am->channel_cfg = GF_AUDIO_CH_FRONT_LEFT | GF_AUDIO_CH_FRONT_RIGHT; else am->channel_cfg = GF_AUDIO_CH_FRONT_LEFT; } /*if main mixer recfg output*/ if (am->ar) am->ar->need_reconfig = 1; gf_mixer_lock(am, 0); } Bool gf_mixer_reconfig(GF_AudioMixer *am) { u32 i, count, numInit, max_sample_rate, max_channels, max_bps, cfg_changed, ch_cfg; gf_mixer_lock(am, 1); if (am->isEmpty || !am->must_reconfig) { gf_mixer_lock(am, 0); return 0; } numInit = 0; max_sample_rate = am->sample_rate; max_channels = am->nb_channels; max_bps = am->bits_per_sample; cfg_changed = 0; ch_cfg = 0; max_sample_rate = 0, count = gf_list_count(am->sources); assert(count); for (i=0; isources, i); has_cfg = in->src->GetConfig(in->src, 1); if (has_cfg) { /*check same cfg...*/ if (in->src->sr * in->src->chan * in->src->bps == 8*in->bytes_per_sec) { numInit++; continue; } } else continue; /*update out cfg*/ if ((count==1) && (max_sample_rate != in->src->sr)) { // cfg_changed = 1; max_sample_rate = in->src->sr; } else if (max_sample_ratesrc->sr) { // cfg_changed = 1; max_sample_rate = in->src->sr; } if ((count==1) && (max_bps!=in->src->bps)) { cfg_changed = 1; max_bps = in->src->bps; } else if (max_bpssrc->bps) { cfg_changed = 1; max_bps = in->src->bps; } if (!am->force_channel_out) { if ((count==1) && (max_channels!=in->src->chan)) { cfg_changed = 1; max_channels = in->src->chan; if (in->src->chan>2) ch_cfg |= in->src->ch_cfg; } else if (max_channels < in->src->chan) { cfg_changed = 1; max_channels = in->src->chan; if (in->src->chan>2) ch_cfg |= in->src->ch_cfg; } } numInit++; in->bytes_per_sec = in->src->sr * in->src->chan * in->src->bps / 8; /*cfg has changed, we must reconfig everything*/ if (cfg_changed || (max_sample_rate != am->sample_rate) ) { in->has_prev = 0; memset(&in->last_channels, 0, sizeof(s16)*GF_SR_MAX_CHANNELS); } } if (cfg_changed || (max_sample_rate && (max_sample_rate != am->sample_rate)) ) { if (max_channels>2) { if (ch_cfg != am->channel_cfg) { /*recompute num channel based on all input channels*/ max_channels = 0; if (ch_cfg & GF_AUDIO_CH_FRONT_LEFT) max_channels ++; if (ch_cfg & GF_AUDIO_CH_FRONT_RIGHT) max_channels ++; if (ch_cfg & GF_AUDIO_CH_FRONT_CENTER) max_channels ++; if (ch_cfg & GF_AUDIO_CH_LFE) max_channels ++; if (ch_cfg & GF_AUDIO_CH_BACK_LEFT) max_channels ++; if (ch_cfg & GF_AUDIO_CH_BACK_RIGHT) max_channels ++; if (ch_cfg & GF_AUDIO_CH_BACK_CENTER) max_channels ++; if (ch_cfg & GF_AUDIO_CH_SIDE_LEFT) max_channels ++; if (ch_cfg & GF_AUDIO_CH_SIDE_RIGHT) max_channels ++; } } else { ch_cfg = GF_AUDIO_CH_FRONT_LEFT; if (max_channels==2) ch_cfg |= GF_AUDIO_CH_FRONT_RIGHT; } gf_mixer_set_config(am, max_sample_rate, max_channels, max_bps, ch_cfg); } if (numInit == count) am->must_reconfig = 0; gf_mixer_lock(am, 0); return cfg_changed; } static GFINLINE u32 get_channel_out_pos(u32 in_ch, u32 out_cfg) { u32 i, cfg, pos; pos = 0; for (i=0; i<9; i++) { cfg = 1<<(i); if (out_cfg & cfg) { if (cfg == in_ch) return pos; pos++; } } return GF_SR_MAX_CHANNELS; } /*this is crude, we'd need a matrix or something*/ static GFINLINE void am_map_channels(s32 *inChan, u32 nb_in, u32 in_cfg, u32 nb_out, u32 out_cfg) { u32 i; if (nb_in==1) { /*mono to stereo*/ if (nb_out==2) { inChan[1] = inChan[0]; } else if (nb_out>2) { /*if center channel use it (we assume we always have stereo channels)*/ if (out_cfg & GF_AUDIO_CH_FRONT_CENTER) { inChan[2] = inChan[0]; inChan[0] = 0; for (i=3; i>=1; /*done*/ if (ch==10) return; } pos = get_channel_out_pos((1<>=1; } for (i=nb_in; inb_out) { s32 bckup[GF_SR_MAX_CHANNELS]; u32 pos; u32 cfg = in_cfg; u32 ch = 0; memcpy(bckup, inChan, sizeof(s32)*nb_in); for (i=0; i>=1; /*done*/ if (ch==10) return; } pos = get_channel_out_pos( (1<>=1; } } } static void AM_GetInputData(GF_AudioMixer *am, MixerInput *in, u32 audio_delay) { u32 i, j, in_ch, out_ch, prev, next, src_samp, ratio, src_size; Bool use_prev; s16 *in_s16; s8 *in_s8; s32 frac, inChan[GF_SR_MAX_CHANNELS], inChanNext[GF_SR_MAX_CHANNELS]; in_s8 = (s8 *) in->src->FetchFrame(in->src->callback, &src_size, audio_delay); if (!in_s8) { in->has_prev = 0; /*done, stop fill*/ in->out_samples_to_write = 0; return; } ratio = (u32) (in->src->sr * FIX2INT(255*in->speed) / am->sample_rate); src_samp = (u32) (src_size * 8 / in->src->bps / in->src->chan); in_ch = in->src->chan; out_ch = am->nb_channels; if (in->src->bps == 8) { in_s16 = NULL; } else { in_s16 = (s16 *) in_s8; in_s8 = NULL; } /*just in case, if only 1 sample available in src, copy over and discard frame since we cannot interpolate audio*/ if (src_samp==1) { in->has_prev = 1; for (j=0; jlast_channels[j] = in_s16 ? in_s16[j] : in_s8[j]; in->in_bytes_used = src_size; return; } /*while space to fill and input data, convert*/ use_prev = in->has_prev; i = 0; next = prev = 0; while (1) { prev = (u32) (i*ratio + 1) / 255; if (prev>=src_samp) break; next = prev+1; frac = (i*ratio) - 255*prev; if (frac && (next==src_samp)) break; if (use_prev && prev) use_prev = 0; if (in_s16) { for (j=0; jlast_channels[j] : in_s16[in_ch*prev + j]; inChanNext[j] = in_s16[in_ch*next + j]; inChan[j] = (frac*inChanNext[j] + (255-frac)*inChan[j]) / 255; } } else { for (j=0; jlast_channels[j] : in_s8[in_ch*prev + j]; inChanNext[j] = in_s8[in_ch*next + j]; inChan[j] = (frac*inChanNext[j] + (255-frac)*inChan[j]) / 255; } } am_map_channels(inChan, in_ch, in->src->ch_cfg, out_ch, am->channel_cfg); for (j=0; jch_buf[j] + in->out_samples_written) = (s32) (inChan[j] * FIX2INT(100*in->pan[j]) / 100 ); } in->out_samples_written ++; if (in->out_samples_written == in->out_samples_to_write) break; i++; } if (!(ratio%255)) { in->has_prev = 0; if (next==src_samp) { in->in_bytes_used = src_size; } else { in->in_bytes_used = MIN(src_size, prev*in->src->bps * in->src->chan / 8); } } else { in->has_prev = 1; if (next==src_samp) { for (j=0; jlast_channels[j] = inChanNext[j]; in->in_bytes_used = src_size; } else { in->in_bytes_used = prev*in->src->bps * in->src->chan / 8; if (in->in_bytes_used>src_size) { in->in_bytes_used = src_size; for (j=0; jlast_channels[j] = inChanNext[j]; } else { for (j=0; jlast_channels[j] = in_s16 ? in_s16[in_ch*prev + j] : in_s8[in_ch*prev + j]; } } } /*cf below, make sure we call release*/ in->in_bytes_used += 1; } u32 gf_mixer_get_output(GF_AudioMixer *am, void *buffer, u32 buffer_size) { MixerInput *in, *single_source; Fixed pan[6]; Bool is_muted; u32 i, j, count, size, in_size, nb_samples, delay, nb_written; s32 *out_mix, nb_act_src; char *data, *ptr; /*the config has changed we don't write to output since settings change*/ if (gf_mixer_reconfig(am)) return 0; gf_mixer_lock(am, 1); count = gf_list_count(am->sources); if (!count) { memset(buffer, 0, buffer_size); gf_mixer_lock(am, 0); return 0; } delay = 0; if (am->ar && !(am->ar->flags & GF_SR_AUDIO_NO_RESYNC) ) delay = am->ar->audio_delay; single_source = NULL; if (count!=1) goto do_mix; if (am->force_channel_out) goto do_mix; single_source = (MixerInput *) gf_list_get(am->sources, 0); /*if cfg changed or unknown return*/ if (!single_source->src->GetConfig(single_source->src, 0)) { am->must_reconfig = 1; gf_mixer_reconfig(am); memset(buffer, 0, buffer_size); gf_mixer_lock(am, 0); return 0; } /*this happens if input SR cannot be mapped to output audio hardware*/ if (single_source->src->sr != am->sample_rate) goto do_mix; /*note we don't check output cfg: if the number of channel is the same then the channel cfg is the same*/ if (single_source->src->chan != am->nb_channels) goto do_mix; if (single_source->src->GetSpeed(single_source->src->callback)!=FIX_ONE) goto do_mix; if (single_source->src->GetChannelVolume(single_source->src->callback, pan)) goto do_mix; single_source_mix: ptr = (char *)buffer; in_size = buffer_size; is_muted = single_source->src->IsMuted(single_source->src->callback); while (buffer_size) { data = single_source->src->FetchFrame(single_source->src->callback, &size, delay); if (!data || !size) break; /*don't copy more than possible*/ if (size > buffer_size) size = buffer_size; if (is_muted) { memset(ptr, 0, size); } else { memcpy(ptr, data, size); } buffer_size -= size; ptr += size; single_source->src->ReleaseFrame(single_source->src->callback, size); delay = 0; } /*not completely filled*/ if (buffer_size) { if (!data) { GF_LOG(GF_LOG_WARNING, GF_LOG_RENDER, ("[Audio Mixer] not enough input data (%d still to fill)\n", buffer_size)); } memset(ptr, 0, buffer_size); } gf_mixer_lock(am, 0); return (in_size - buffer_size); do_mix: nb_act_src = 0; nb_samples = buffer_size / (am->nb_channels * am->bits_per_sample / 8); /*step 1, cfg*/ if (am->output_sizeoutput) free(am->output); am->output = (s32*)malloc(sizeof(s32) * buffer_size); am->output_size = buffer_size; } single_source = NULL; for (i=0; isources, i); if (in->src->IsMuted(in->src->callback)) continue; if (in->buffer_size < nb_samples) { for (j=0; jch_buf[j]) free(in->ch_buf[j]); in->ch_buf[j] = (s32 *) malloc(sizeof(s32) * nb_samples); } in->buffer_size = nb_samples; } in->speed = in->src->GetSpeed(in->src->callback); if (in->speed<0) in->speed *= -1; in->out_samples_written = 0; in->in_bytes_used = 0; /*if cfg unknown or changed (AudioBuffer child...) invalidate cfg settings*/ if (!in->src->GetConfig(in->src, 0)) { nb_act_src = 0; am->must_reconfig = 1; /*if main mixer reconfig asap*/ if (am->ar) gf_mixer_reconfig(am); break; } else if (in->speed==0) { in->out_samples_to_write = 0; } else { assert(in->src->sr); in->out_samples_to_write = nb_samples; if (in->src->IsMuted(in->src->callback)) { memset(in->pan, 0, sizeof(Fixed)*6); } else { if (!in->src->GetChannelVolume(in->src->callback, in->pan)) { /*track first active source with same cfg as mixer*/ if (!single_source && (in->src->sr == am->sample_rate) && (in->src->chan == am->nb_channels) && (in->speed == FIX_ONE) ) single_source = in; } } nb_act_src ++; } } if (!nb_act_src) { memset(buffer, 0, sizeof(char)*buffer_size); gf_mixer_lock(am, 0); return 0; } /*if only one active source in native format, process as single source (direct copy) this is needed because mediaControl on an audio object doesn't deactivate it (eg the audio object is still present in the mixer). this opt is typically usefull for language selection content (cf mp4menu)*/ if ((nb_act_src==1) && single_source) goto single_source_mix; /*step 2, fill all buffers*/ while (nb_act_src) { u32 nb_to_fill = 0; /*fill*/ for (i=0; isources, i); if (in->out_samples_to_write>in->out_samples_written) { AM_GetInputData(am, in, in->out_samples_written ? 0 : delay); if (in->out_samples_to_write>in->out_samples_written) nb_to_fill++; } } /*release - this is done in 2 steps in case 2 audio object use the same source...*/ for (i=0; isources, i); if (in->in_bytes_used) in->src->ReleaseFrame(in->src->callback, in->in_bytes_used-1); in->in_bytes_used = 0; } if (!nb_to_fill) break; } /*step 3, mix the final buffer*/ memset(am->output, 0, sizeof(s32) * buffer_size); nb_written = 0; for (i=0; ioutput; in = (MixerInput *)gf_list_get(am->sources, i); if (!in->out_samples_to_write) continue; /*only write what has been filled in the source buffer (may be less than output size)*/ for (j=0; jout_samples_written; j++) { u32 k; for (k=0; knb_channels; k++) { (*out_mix) += * (in->ch_buf[k] + j); out_mix += 1; } } if (nb_written < in->out_samples_written) nb_written = in->out_samples_written; } if (!nb_written) { memset(buffer, 0, sizeof(char)*buffer_size); gf_mixer_lock(am, 0); return 0; } out_mix = am->output; if (am->bits_per_sample==16) { s16 *out_s16 = (s16 *) buffer; for (i=0; inb_channels; j++) { s32 samp = (*out_mix / nb_act_src); if (samp > GF_SHORT_MAX) samp = GF_SHORT_MAX; else if (samp < GF_SHORT_MIN) samp = GF_SHORT_MIN; (*out_s16) = samp; out_s16 += 1; out_mix += 1; } } } else { s8 *out_s8 = (s8 *) buffer; for (i=0; inb_channels; j++) { s32 samp = (*out_mix / nb_act_src); if (samp > 127) samp = 127; else if (samp < -128) samp = -128; (*out_s8) = samp; out_s8 += 1; out_mix += 1; } } } nb_written *= am->nb_channels*am->bits_per_sample/8; if (buffer_size > nb_written) memset((char *)buffer + nb_written, 0, sizeof(char)*(buffer_size-nb_written)); gf_mixer_lock(am, 0); return nb_written; }