diff --git a/api/iOS/VCSimpleSession.mm b/api/iOS/VCSimpleSession.mm index 5ec32b55..ebaf5034 100644 --- a/api/iOS/VCSimpleSession.mm +++ b/api/iOS/VCSimpleSession.mm @@ -1,18 +1,18 @@ /* - + Video Core Copyright (c) 2014 James G. Hurley - + Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - + The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE @@ -20,7 +20,7 @@ of this software and associated documentation files (the "Software"), to deal LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - + */ #import @@ -61,25 +61,25 @@ of this software and associated documentation files (the "Software"), to deal static const int kMinVideoBitrate = 32000; namespace videocore { namespace simpleApi { - + using PixelBufferCallback = std::function ; - + class PixelBufferOutput : public IOutput { public: PixelBufferOutput(PixelBufferCallback callback) : m_callback(callback) {}; - + void pushBuffer(const uint8_t* const data, size_t size, IMetadata& metadata) { m_callback(data, size); } - + private: - + PixelBufferCallback m_callback; }; } @@ -87,9 +87,9 @@ void pushBuffer(const uint8_t* const data, @interface VCSimpleSession() { - + VCPreviewView* _previewView; - + std::shared_ptr m_pbOutput; std::shared_ptr m_micSource; std::shared_ptr m_cameraSource; @@ -107,37 +107,37 @@ @interface VCSimpleSession() std::shared_ptr m_aacEncoder; std::shared_ptr m_h264Packetizer; std::shared_ptr m_aacPacketizer; - + std::shared_ptr m_aacSplit; std::shared_ptr m_h264Split; std::shared_ptr m_muxer; - + std::shared_ptr m_outputSession; - - + + // properties - + dispatch_queue_t _graphManagementQueue; - + CGSize _videoSize; int _bitrate; - + int _fps; int _bpsCeiling; int _estimatedThroughput; - + BOOL _useInterfaceOrientation; float _videoZoomFactor; int _audioChannelCount; float _audioSampleRate; float _micGain; - + VCCameraState _cameraState; VCAspectMode _aspectMode; VCSessionState _rtmpSessionState; BOOL _orientationLocked; BOOL _torch; - + BOOL _useAdaptiveBitrate; BOOL _continuousAutofocus; BOOL _continuousExposure; @@ -292,7 +292,7 @@ - (void) setVideoZoomFactor:(float)videoZoomFactor - (void) setAudioChannelCount:(int)channelCount { _audioChannelCount = MIN(2, MAX(channelCount,2)); // We can only support a channel count of 2 with AAC - + if(m_audioMixer) { m_audioMixer->setChannelCount(channelCount); } @@ -303,7 +303,7 @@ - (int) audioChannelCount } - (void) setAudioSampleRate:(float)sampleRate { - + _audioSampleRate = (sampleRate > 33075 ? 44100 : 22050); // We can only support 44100 / 22050 with AAC + RTMP if(m_audioMixer) { m_audioMixer->setFrequencyInHz(sampleRate); @@ -349,7 +349,7 @@ - (void) setContinuousExposure:(BOOL)continuousExposure - (void) setFocusPointOfInterest:(CGPoint)focusPointOfInterest { _focusPOI = focusPointOfInterest; - + if(m_cameraSource) { m_cameraSource->setFocusPointOfInterest(focusPointOfInterest.x, focusPointOfInterest.y); } @@ -395,7 +395,7 @@ - (instancetype) initWithVideoSize:(CGSize)videoSize useInterfaceOrientation:NO cameraState:VCCameraStateBack aspectMode:VCAspectModeFit]; - + } return self; } @@ -472,18 +472,18 @@ - (void) initInternalWithVideoSize:(CGSize)videoSize self.audioSampleRate = 44100.; self.useAdaptiveBitrate = NO; self.aspectMode = aspectMode; - + _previewView = [[VCPreviewView alloc] init]; self.videoZoomFactor = 1.f; - + _cameraState = cameraState; _exposurePOI = _focusPOI = CGPointMake(0.5f, 0.5f); _continuousExposure = _continuousAutofocus = YES; - + _graphManagementQueue = dispatch_queue_create("com.videocore.session.graph", 0); - + __block VCSimpleSession* bSelf = self; - + dispatch_async(_graphManagementQueue, ^{ [bSelf setupGraph]; }); @@ -502,18 +502,18 @@ - (void) dealloc m_pbOutput.reset(); [_previewView release]; _previewView = nil; - + dispatch_release(_graphManagementQueue); - + [super dealloc]; } - (void) startRtmpSessionWithURL:(NSString *)rtmpUrl andStreamKey:(NSString *)streamKey { - + __block VCSimpleSession* bSelf = self; - + dispatch_async(_graphManagementQueue, ^{ [bSelf startSessionInternal:rtmpUrl streamKey:streamKey]; }); @@ -523,34 +523,34 @@ - (void) startSessionInternal: (NSString*) rtmpUrl { std::stringstream uri ; uri << (rtmpUrl ? [rtmpUrl UTF8String] : "") << "/" << (streamKey ? [streamKey UTF8String] : ""); - + m_outputSession.reset( new videocore::RTMPSession ( uri.str(), [=](videocore::RTMPSession& session, ClientState_t state) { - + DLog("ClientState: %d\n", state); - + switch(state) { - + case kClientStateConnected: self.rtmpSessionState = VCSessionStateStarting; break; case kClientStateSessionStarted: { - + __block VCSimpleSession* bSelf = self; dispatch_async(_graphManagementQueue, ^{ [bSelf addEncodersAndPacketizers]; }); } self.rtmpSessionState = VCSessionStateStarted; - + break; case kClientStateError: self.rtmpSessionState = VCSessionStateError; - [self endRtmpSession]; - self->m_outputSession.reset(); + // [self endRtmpSession]; + // self->m_outputSession.reset(); break; case kClientStateNotConnected: self.rtmpSessionState = VCSessionStateEnded; @@ -558,26 +558,26 @@ - (void) startSessionInternal: (NSString*) rtmpUrl break; default: break; - + } - + }) ); VCSimpleSession* bSelf = self; - + _bpsCeiling = _bitrate; - + if ( self.useAdaptiveBitrate ) { _bitrate = 500000; } - + m_outputSession->setBandwidthCallback([=](float vector, float predicted, int inst) { - + bSelf->_estimatedThroughput = predicted; auto video = std::dynamic_pointer_cast( bSelf->m_h264Encoder ); auto audio = std::dynamic_pointer_cast( bSelf->m_aacEncoder ); if(video && audio && bSelf.useAdaptiveBitrate) { - + if ([bSelf.delegate respondsToSelector:@selector(detectedThroughput:)]) { [bSelf.delegate detectedThroughput:predicted]; } @@ -585,17 +585,17 @@ - (void) startSessionInternal: (NSString*) rtmpUrl [bSelf.delegate detectedThroughput:predicted videoRate:video->bitrate()]; } - + int videoBr = 0; - + if(vector != 0) { - + vector = vector < 0 ? -1 : 1 ; - + videoBr = video->bitrate(); - + if (audio) { - + if ( videoBr > 500000 ) { audio->setBitrate(128000); } else if (videoBr <= 500000 && videoBr > 250000) { @@ -604,8 +604,8 @@ - (void) startSessionInternal: (NSString*) rtmpUrl audio->setBitrate(80000); } } - - + + if(videoBr > 1152000) { video->setBitrate(std::min(int((videoBr / 384000 + vector )) * 384000, bSelf->_bpsCeiling) ); } @@ -620,36 +620,39 @@ - (void) startSessionInternal: (NSString*) rtmpUrl } DLog("\n(%f) AudioBR: %d VideoBR: %d (%f)\n", vector, audio->bitrate(), video->bitrate(), predicted); } /* if(vector != 0) */ - + } /* if(video && audio && m_adaptiveBREnabled) */ - - + + }); - + videocore::RTMPSessionParameters_t sp ( 0. ); - + sp.setData(self.videoSize.width, self.videoSize.height, 1. / static_cast(self.fps), self.bitrate, self.audioSampleRate, (self.audioChannelCount == 2)); - + m_outputSession->setSessionParameters(sp); } - (void) endRtmpSession { - + m_h264Packetizer.reset(); m_aacPacketizer.reset(); - m_videoSplit->removeOutput(m_h264Encoder); + if (m_videoSplit != nullptr) { + m_videoSplit->removeOutput(m_h264Encoder); + } + m_h264Encoder.reset(); m_aacEncoder.reset(); - + m_outputSession.reset(); - + _bitrate = _bpsCeiling; - + self.rtmpSessionState = VCSessionStateEnded; } - (void) getCameraPreviewLayer:(AVCaptureVideoPreviewLayer **)previewLayer { @@ -660,35 +663,38 @@ - (void) getCameraPreviewLayer:(AVCaptureVideoPreviewLayer **)previewLayer { //Set property filter for the new enum + set dynamically the sourceFilter for the video mixer - (void)setFilter:(VCFilter)filterToChange { - NSString *filterName = @"com.videocore.filters.bgra"; - - switch (filterToChange) { - case VCFilterNormal: - filterName = @"com.videocore.filters.bgra"; - break; - case VCFilterGray: - filterName = @"com.videocore.filters.grayscale"; - break; - case VCFilterInvertColors: - filterName = @"com.videocore.filters.invertColors"; - break; - case VCFilterSepia: - filterName = @"com.videocore.filters.sepia"; - break; - case VCFilterFisheye: - filterName = @"com.videocore.filters.fisheye"; - break; - case VCFilterGlow: - filterName = @"com.videocore.filters.glow"; - break; - default: - break; - } - - _filter = filterToChange; - NSLog(@"FILTER IS : [%d]", (int)_filter); - std::string convertString([filterName UTF8String]); + NSString *filterName = @"com.videocore.filters.bgra"; + + switch (filterToChange) { + case VCFilterNormal: + filterName = @"com.videocore.filters.bgra"; + break; + case VCFilterGray: + filterName = @"com.videocore.filters.grayscale"; + break; + case VCFilterInvertColors: + filterName = @"com.videocore.filters.invertColors"; + break; + case VCFilterSepia: + filterName = @"com.videocore.filters.sepia"; + break; + case VCFilterFisheye: + filterName = @"com.videocore.filters.fisheye"; + break; + case VCFilterGlow: + filterName = @"com.videocore.filters.glow"; + break; + default: + break; + } + + _filter = filterToChange; + NSLog(@"FILTER IS : [%d]", (int)_filter); + std::string convertString([filterName UTF8String]); + + if (m_videoMixer != nullptr) { m_videoMixer->setSourceFilter(m_cameraSource, dynamic_cast(m_videoMixer->filterFactory().filter(convertString))); // default is com.videocore.filters.bgra + } } // ----------------------------------------------------------------------------- @@ -700,38 +706,53 @@ - (void)setFilter:(VCFilter)filterToChange { - (void) setupGraph { const double frameDuration = 1. / static_cast(self.fps); - + { // Add audio mixer const double aacPacketTime = 1024. / self.audioSampleRate; - + + if (!self.audioChannelCount) { + self.audioChannelCount = 2; + } + + if (!self.audioSampleRate) { + self.audioSampleRate = 44100; + } + m_audioMixer = std::make_shared(self.audioChannelCount, self.audioSampleRate, 16, aacPacketTime); - - + + // The H.264 Encoder introduces about 2 frames of latency, so we will set the minimum audio buffer duration to 2 frames. - m_audioMixer->setMinimumBufferDuration(frameDuration*2); + if (m_audioMixer != nullptr) { + m_audioMixer->setMinimumBufferDuration(frameDuration*2); + + } } #ifdef __APPLE__ #ifdef TARGET_OS_IPHONE - - + + { // Add video mixer + NSLog(@">>>>>>>> Video size width = %f, height = %f", self.videoSize.width, self.videoSize.height); + + m_videoMixer = std::make_shared(self.videoSize.width, self.videoSize.height, frameDuration); - + + } - + { auto videoSplit = std::make_shared(); - + m_videoSplit = videoSplit; VCPreviewView* preview = (VCPreviewView*)self.previewView; - + m_pbOutput = std::make_shared([=](const void* const data, size_t size){ CVPixelBufferRef ref = (CVPixelBufferRef)data; [preview drawFrame:ref]; @@ -739,62 +760,89 @@ - (void) setupGraph self.rtmpSessionState = VCSessionStatePreviewStarted; } }); - + videoSplit->setOutput(m_pbOutput); - + m_videoMixer->setOutput(videoSplit); - + } - + #else #endif // TARGET_OS_IPHONE #endif // __APPLE__ - + // Create sources { // Add camera source m_cameraSource = std::make_shared(); m_cameraSource->setOrientationLocked(self.orientationLocked); auto aspectTransform = std::make_shared(self.videoSize.width,self.videoSize.height,m_aspectMode); - + auto positionTransform = std::make_shared(self.videoSize.width/2, self.videoSize.height/2, self.videoSize.width * self.videoZoomFactor, self.videoSize.height * self.videoZoomFactor, self.videoSize.width, self.videoSize.height ); - - + + std::dynamic_pointer_cast(m_cameraSource)->setupCamera(self.fps,(self.cameraState == VCCameraStateFront),self.useInterfaceOrientation,nil,^{ - m_cameraSource->setContinuousAutofocus(true); - m_cameraSource->setContinuousExposure(true); - - m_cameraSource->setOutput(aspectTransform); - - m_videoMixer->setSourceFilter(m_cameraSource, dynamic_cast(m_videoMixer->filterFactory().filter("com.videocore.filters.bgra"))); - _filter = VCFilterNormal; - aspectTransform->setOutput(positionTransform); - positionTransform->setOutput(m_videoMixer); - m_aspectTransform = aspectTransform; - m_positionTransform = positionTransform; - - // Inform delegate that camera source has been added - if ([_delegate respondsToSelector:@selector(didAddCameraSource:)]) { - [_delegate didAddCameraSource:self]; + + + + if (m_cameraSource != nullptr) { + m_cameraSource->setContinuousAutofocus(true); + m_cameraSource->setContinuousExposure(true); + + m_cameraSource->setOutput(aspectTransform); + + + if (m_videoMixer != nullptr) { + m_videoMixer->setSourceFilter(m_cameraSource, dynamic_cast(m_videoMixer->filterFactory().filter("com.videocore.filters.bgra"))); + _filter = VCFilterNormal; + aspectTransform->setOutput(positionTransform); + positionTransform->setOutput(m_videoMixer); + m_aspectTransform = aspectTransform; + m_positionTransform = positionTransform; + } + + // Inform delegate that camera source has been added + if ([_delegate respondsToSelector:@selector(didAddCameraSource:)]) { + [_delegate didAddCameraSource:self]; + } } }); } { - // Add mic source - m_micSource = std::make_shared(self.audioSampleRate, self.audioChannelCount); - m_micSource->setOutput(m_audioMixer); - - const auto epoch = std::chrono::steady_clock::now(); - - m_audioMixer->setEpoch(epoch); - m_videoMixer->setEpoch(epoch); - - m_audioMixer->start(); - m_videoMixer->start(); - + if (m_videoMixer != nullptr && m_audioMixer != nullptr) { + + // Add mic source + if (!self.audioChannelCount) { + self.audioChannelCount = 2; + } + + if (!self.audioSampleRate) { + self.audioSampleRate = 44100; + } + + m_micSource = std::make_shared(self.audioSampleRate, self.audioChannelCount); + + if (m_micSource != nullptr && m_audioMixer != nullptr) { + m_micSource->setOutput(m_audioMixer); + } + + const auto epoch = std::chrono::steady_clock::now(); + if (m_audioMixer != nullptr) { + m_audioMixer->setEpoch(epoch); + } + + if (m_videoMixer != nullptr) { + m_videoMixer->setEpoch(epoch); + } + + if (m_audioMixer != nullptr && m_videoMixer != nullptr) { + m_audioMixer->start(); + m_videoMixer->start(); + } + } } } - (void) addEncodersAndPacketizers @@ -802,7 +850,7 @@ - (void) addEncodersAndPacketizers int ctsOffset = 2000 / self.fps; // 2 * frame duration { // Add encoders - + m_aacEncoder = std::make_shared(self.audioSampleRate, self.audioChannelCount, 96000); if(SYSTEM_VERSION_GREATER_THAN_OR_EQUAL_TO(@"8.0")) { // If >= iOS 8.0 use the VideoToolbox encoder that does not write to disk. @@ -828,15 +876,15 @@ - (void) addEncodersAndPacketizers m_h264Split = std::make_shared(); m_aacEncoder->setOutput(m_aacSplit); m_h264Encoder->setOutput(m_h264Split); - + } { m_h264Packetizer = std::make_shared(ctsOffset); m_aacPacketizer = std::make_shared(self.audioSampleRate, self.audioChannelCount, ctsOffset); - + m_h264Split->setOutput(m_h264Packetizer); m_aacSplit->setOutput(m_aacPacketizer); - + } { /*m_muxer = std::make_shared(); @@ -847,22 +895,62 @@ - (void) addEncodersAndPacketizers m_aacSplit->setOutput(m_muxer); m_h264Split->setOutput(m_muxer);*/ } - - + + m_h264Packetizer->setOutput(m_outputSession); m_aacPacketizer->setOutput(m_outputSession); - + } /* -- (void) addPixelBufferSource: (UIImage*) image - withRect:(CGRect)rect { + - (void) addPixelBufferSource: (UIImage*) image + withRect:(CGRect)rect { + CGImageRef ref = [image CGImage]; + + m_pixelBufferSource = std::make_shared(CGImageGetWidth(ref), + CGImageGetHeight(ref), + 'BGRA'); + + NSUInteger width = CGImageGetWidth(ref); + NSUInteger height = CGImageGetHeight(ref); + CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB(); + unsigned char *rawData = (unsigned char*) calloc(height * width * 4, sizeof(unsigned char)); + NSUInteger bytesPerPixel = 4; + NSUInteger bytesPerRow = bytesPerPixel * width; + NSUInteger bitsPerComponent = 8; + CGContextRef context = CGBitmapContextCreate(rawData, width, height, + bitsPerComponent, bytesPerRow, colorSpace, + kCGImageAlphaPremultipliedFirst | kCGBitmapByteOrder32Little); + CGColorSpaceRelease(colorSpace); + + CGContextDrawImage(context, CGRectMake(0, 0, width, height), ref); + CGContextRelease(context); + + m_pbAspect = std::make_shared(rect.size.width,rect.size.height,videocore::AspectTransform::kAspectFit); + + m_pbPosition = std::make_shared(rect.origin.x, rect.origin.y, + rect.size.width, rect.size.height, + self.videoSize.width, self.videoSize.height + ); + m_pixelBufferSource->setOutput(m_pbAspect); + m_pbAspect->setOutput(m_pbPosition); + m_pbPosition->setOutput(m_videoMixer); + m_videoMixer->registerSource(m_pixelBufferSource); + m_pixelBufferSource->pushPixelBuffer(rawData, width * height * 4); + + free(rawData); + + }*/ + +- (void) addPixelBufferSource: (UIImage*) image withRect:(CGRect)rect { CGImageRef ref = [image CGImage]; + std::shared_ptr temp_pixelBufferSource = m_pixelBufferSource; + + m_pixelBufferSource = std::make_shared(CGImageGetWidth(ref), CGImageGetHeight(ref), 'BGRA'); - NSUInteger width = CGImageGetWidth(ref); NSUInteger height = CGImageGetHeight(ref); CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB(); @@ -883,61 +971,20 @@ - (void) addPixelBufferSource: (UIImage*) image m_pbPosition = std::make_shared(rect.origin.x, rect.origin.y, rect.size.width, rect.size.height, self.videoSize.width, self.videoSize.height - ); - m_pixelBufferSource->setOutput(m_pbAspect); - m_pbAspect->setOutput(m_pbPosition); - m_pbPosition->setOutput(m_videoMixer); - m_videoMixer->registerSource(m_pixelBufferSource); - m_pixelBufferSource->pushPixelBuffer(rawData, width * height * 4); - - free(rawData); + ); -}*/ - -- (void) addPixelBufferSource: (UIImage*) image -withRect:(CGRect)rect { -CGImageRef ref = [image CGImage]; - -std::shared_ptr temp_pixelBufferSource = m_pixelBufferSource; - - -m_pixelBufferSource = std::make_shared(CGImageGetWidth(ref), -CGImageGetHeight(ref), -'BGRA'); -NSUInteger width = CGImageGetWidth(ref); -NSUInteger height = CGImageGetHeight(ref); -CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB(); -unsigned char *rawData = (unsigned char*) calloc(height * width * 4, sizeof(unsigned char)); -NSUInteger bytesPerPixel = 4; -NSUInteger bytesPerRow = bytesPerPixel * width; -NSUInteger bitsPerComponent = 8; -CGContextRef context = CGBitmapContextCreate(rawData, width, height, -bitsPerComponent, bytesPerRow, colorSpace, -kCGImageAlphaPremultipliedFirst | kCGBitmapByteOrder32Little); -CGColorSpaceRelease(colorSpace); - -CGContextDrawImage(context, CGRectMake(0, 0, width, height), ref); -CGContextRelease(context); - -m_pbAspect = std::make_shared(rect.size.width,rect.size.height,videocore::AspectTransform::kAspectFit); - -m_pbPosition = std::make_shared(rect.origin.x, rect.origin.y, -rect.size.width, rect.size.height, -self.videoSize.width, self.videoSize.height -); -m_pixelBufferSource->setOutput(m_pbAspect); -m_pbAspect->setOutput(m_pbPosition); -m_pbPosition->setOutput(m_videoMixer); -m_videoMixer->registerSource(m_pixelBufferSource); -m_pixelBufferSource->pushPixelBuffer(rawData, width * height * 4); - -free(rawData); - - -if(temp_pixelBufferSource){ -m_videoMixer->unregisterSource(temp_pixelBufferSource); -} - + if (m_videoMixer != nullptr && m_audioMixer != nullptr) { + m_pixelBufferSource->setOutput(m_pbAspect); + m_pbAspect->setOutput(m_pbPosition); + m_pbPosition->setOutput(m_videoMixer); + m_videoMixer->registerSource(m_pixelBufferSource); + m_pixelBufferSource->pushPixelBuffer(rawData, width * height * 4); + + free(rawData); + if(temp_pixelBufferSource){ + m_videoMixer->unregisterSource(temp_pixelBufferSource); + } + } } - (NSString *) applicationDocumentsDirectory @@ -946,4 +993,4 @@ - (NSString *) applicationDocumentsDirectory NSString *basePath = ([paths count] > 0) ? [paths objectAtIndex:0] : nil; return basePath; } -@end +@end \ No newline at end of file