Unverified Commit 91ba07c9 authored by Hampton Maxwell's avatar Hampton Maxwell Committed by GitHub
Browse files

Merge pull request #1109 from nfb-onf/master 

iOS Side loading for captions and offline support
parents e5ab177a fe26f54b

android-exoplayer/src/main/java/com/brentvatne/exoplayer/ReactExoplayerView.java

+23 −4
Changes for android-exoplayer/src/main/java/com/brentvatne/exoplayer/ReactExoplayerView.java: 23 added lines, 4 removed lines.
Original line number Diff line number Diff line
@@ -11,6 +11,7 @@ import android.text.TextUtils; 
import android.util.Log;

import android.view.View;

import android.view.Window;

import android.view.accessibility.CaptioningManager;

import android.widget.FrameLayout;



import com.brentvatne.react.R;
@@ -68,6 +69,7 @@ import java.lang.Math; 
import java.util.Map;

import java.lang.Object;

import java.util.ArrayList;

import java.util.Locale;



@SuppressLint("ViewConstructor")

class ReactExoplayerView extends FrameLayout implements
@@ -737,10 +739,11 @@ class ReactExoplayerView extends FrameLayout implements 


        TrackGroupArray groups = info.getTrackGroups(index);

        int trackIndex = C.INDEX_UNSET;

        trackSelector.setSelectionOverride(index, groups, null);



        if (TextUtils.isEmpty(type)) {

            // Do nothing

        } else if (type.equals("disabled")) {

            trackSelector.setSelectionOverride(index, groups, null);

            return;

        } else if (type.equals("language")) {

            for (int i = 0; i < groups.length; ++i) {
@@ -760,9 +763,25 @@ class ReactExoplayerView extends FrameLayout implements 
            }

        } else if (type.equals("index")) {

            trackIndex = value.asInt();

        } else { // default. invalid type or "system"

            trackSelector.clearSelectionOverrides(index);

            return;

        } else { // default. Use system settings if possible

            int sdk = android.os.Build.VERSION.SDK_INT;

            if (sdk>18 && groups.length>0) {

                CaptioningManager captioningManager = (CaptioningManager) themedReactContext.getSystemService(Context.CAPTIONING_SERVICE);

                if (captioningManager.isEnabled()) {

                    // default is to take the first object

                    trackIndex = 0;



                    String locale = Locale.getDefault().getDisplayLanguage();

                    for (int i = 0; i < groups.length; ++i) {

                        Format format = groups.get(i).getFormat(0);

                        if (format.language != null && format.language.equals(locale)) {

                            trackIndex = i;

                            break;

                        }

                    }

                }

            } else return;



        }



        if (trackIndex == C.INDEX_UNSET) {

ios/RCTVideo.m

+168 −20
Changes for ios/RCTVideo.m: 168 added lines, 20 removed lines.
Original line number Diff line number Diff line
@@ -3,6 +3,8 @@ 
#import <React/RCTBridgeModule.h>

#import <React/RCTEventDispatcher.h>

#import <React/UIView+React.h>

#include <MediaAccessibility/MediaAccessibility.h>

#include <AVFoundation/AVFoundation.h>



static NSString *const statusKeyPath = @"status";

static NSString *const playbackLikelyToKeepUpKeyPath = @"playbackLikelyToKeepUp";
@@ -43,6 +45,7 @@ static NSString *const timedMetadata = @"timedMetadata"; 
  BOOL _paused;

  BOOL _repeat;

  BOOL _allowsExternalPlayback;

  NSArray * _textTracks;

  NSDictionary * _selectedTextTrack;

  BOOL _playbackStalled;

  BOOL _playInBackground;
@@ -284,6 +287,10 @@ static NSString *const timedMetadata = @"timedMetadata"; 
  [self removePlayerLayer];

  [self removePlayerTimeObserver];

  [self removePlayerItemObservers];

  

  dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(0 * NSEC_PER_SEC)), dispatch_get_main_queue(), ^{

    

    // perform on next run loop, otherwise other passed react-props may not be set

    _playerItem = [self playerItemForSource:source];

    [self addPlayerItemObservers];
@@ -304,7 +311,6 @@ static NSString *const timedMetadata = @"timedMetadata"; 
    

    [self addPlayerTimeObserver];



  dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(0 * NSEC_PER_SEC)), dispatch_get_main_queue(), ^{

    //Perform on next run loop, otherwise onVideoLoadStart is nil

    if(self.onVideoLoadStart) {

      id uri = [source objectForKey:@"uri"];
@@ -316,42 +322,95 @@ static NSString *const timedMetadata = @"timedMetadata"; 
                                        @"target": self.reactTag

                                        });

    }



  });

  _videoLoadStarted = YES;

}



- (NSURL*) urlFilePath:(NSString*) filepath {

  NSArray *paths = NSSearchPathForDirectoriesInDomains(NSDocumentDirectory, NSUserDomainMask, YES);

  

  NSString* relativeFilePath = [filepath lastPathComponent];

  // the file may be multiple levels below the documents directory

  NSArray* fileComponents = [filepath componentsSeparatedByString:@"Documents/"];

  if (fileComponents.count>1) {

    relativeFilePath = [fileComponents objectAtIndex:1];

  }

  

  NSString *path = [paths.firstObject stringByAppendingPathComponent:relativeFilePath];

  if ([[NSFileManager defaultManager] fileExistsAtPath:path]) {

    return [NSURL fileURLWithPath:path];

  }

  return nil;

}



- (AVPlayerItem*)playerItemForSource:(NSDictionary *)source

{

  bool isNetwork = [RCTConvert BOOL:[source objectForKey:@"isNetwork"]];

  bool isAsset = [RCTConvert BOOL:[source objectForKey:@"isAsset"]];

  NSString *uri = [source objectForKey:@"uri"];

  NSString *type = [source objectForKey:@"type"];

  NSDictionary *headers = [source objectForKey:@"requestHeaders"];

  

  NSURL *url = (isNetwork || isAsset) ?

    [NSURL URLWithString:uri] :

    [[NSURL alloc] initFileURLWithPath:[[NSBundle mainBundle] pathForResource:uri ofType:type]];

  AVURLAsset *asset;

  NSMutableDictionary *assetOptions = [[NSMutableDictionary alloc] init];



  if (isNetwork) {

    NSMutableDictionary *assetOptions = [[NSMutableDictionary alloc]init];

    /* Per #1091, this is not a public API. We need to either get approval from Apple to use this

     * or use a different approach.

    NSDictionary *headers = [source objectForKey:@"requestHeaders"];

    if ([headers count] > 0) {

      [assetOptions setObject:headers forKey:@"AVURLAssetHTTPHeaderFieldsKey"];

    }

    */

    NSArray *cookies = [[NSHTTPCookieStorage sharedHTTPCookieStorage] cookies];

    [assetOptions setObject:cookies forKey:AVURLAssetHTTPCookiesKey];

    asset = [AVURLAsset URLAssetWithURL:[NSURL URLWithString:uri] options:assetOptions];

  } else if (isAsset) { //  assets on iOS have to be in the Documents folder

    asset = [AVURLAsset URLAssetWithURL:[self urlFilePath:uri] options:nil];

  } else { // file passed in through JS, or an asset in the Xcode project

    asset = [AVURLAsset URLAssetWithURL:[[NSURL alloc] initFileURLWithPath:[[NSBundle mainBundle] pathForResource:uri ofType:type]] options:nil];

  }

  

    AVURLAsset *asset = [AVURLAsset URLAssetWithURL:url options:assetOptions];

  if (!_textTracks) {

    return [AVPlayerItem playerItemWithAsset:asset];

  }

  else if (isAsset) {

    AVURLAsset *asset = [AVURLAsset URLAssetWithURL:url options:nil];

    return [AVPlayerItem playerItemWithAsset:asset];

  

  // sideload text tracks

  AVMutableComposition *mixComposition = [[AVMutableComposition alloc] init];

  

  AVAssetTrack *videoAsset = [asset tracksWithMediaType:AVMediaTypeVideo].firstObject;

  AVMutableCompositionTrack *videoCompTrack = [mixComposition addMutableTrackWithMediaType:AVMediaTypeVideo preferredTrackID:kCMPersistentTrackID_Invalid];

  [videoCompTrack insertTimeRange:CMTimeRangeMake(kCMTimeZero, videoAsset.timeRange.duration)

                          ofTrack:videoAsset

                           atTime:kCMTimeZero

                            error:nil];



  AVAssetTrack *audioAsset = [asset tracksWithMediaType:AVMediaTypeAudio].firstObject;

  AVMutableCompositionTrack *audioCompTrack = [mixComposition addMutableTrackWithMediaType:AVMediaTypeAudio preferredTrackID:kCMPersistentTrackID_Invalid];

  [audioCompTrack insertTimeRange:CMTimeRangeMake(kCMTimeZero, videoAsset.timeRange.duration)

                          ofTrack:audioAsset

                           atTime:kCMTimeZero

                            error:nil];



  for (int i = 0; i < _textTracks.count; ++i) {

    AVURLAsset *textURLAsset;

    NSString *textUri = [_textTracks objectAtIndex:i][@"uri"];

    if ([[textUri lowercaseString] hasPrefix:@"http"]) {

      textURLAsset = [AVURLAsset URLAssetWithURL:[NSURL URLWithString:textUri] options:assetOptions];

    } else {

      textURLAsset = [AVURLAsset URLAssetWithURL:[self urlFilePath:textUri] options:nil];

    }

    AVAssetTrack *textTrackAsset = [textURLAsset tracksWithMediaType:AVMediaTypeText].firstObject;

    AVMutableCompositionTrack *textCompTrack = [mixComposition

                                                addMutableTrackWithMediaType:AVMediaTypeText

                                                preferredTrackID:kCMPersistentTrackID_Invalid];

    [textCompTrack insertTimeRange:CMTimeRangeMake(kCMTimeZero, videoAsset.timeRange.duration)

                               ofTrack:textTrackAsset

                                atTime:kCMTimeZero

                                 error:nil];

  }



  return [AVPlayerItem playerItemWithURL:url];

  return [AVPlayerItem playerItemWithAsset:mixComposition];

}



- (void)observeValueForKeyPath:(NSString *)keyPath ofObject:(id)object change:(NSDictionary *)change context:(void *)context
@@ -363,7 +422,7 @@ static NSString *const timedMetadata = @"timedMetadata"; 
      if (items && ![items isEqual:[NSNull null]] && items.count > 0) {

        NSMutableArray *array = [NSMutableArray new];

        for (AVMetadataItem *item in items) {

          NSString *value = item.value;

          NSString *value = (NSString *)item.value;

          NSString *identifier = item.identifier;

          

          if (![value isEqual: [NSNull null]]) {
@@ -668,15 +727,91 @@ static NSString *const timedMetadata = @"timedMetadata"; 


- (void)setSelectedTextTrack:(NSDictionary *)selectedTextTrack {

  _selectedTextTrack = selectedTextTrack;

  NSString *type = selectedTextTrack[@"type"];

  if (_textTracks) {

    [self setSideloadedText];

  } else {

    [self setStreamingText];

  }

}



- (void) setSideloadedText {

  NSString *type = _selectedTextTrack[@"type"];

  NSArray* textTracks = [self getTextTrackInfo];

  

  // The first few tracks will be audio & video track

  int firstTextIndex = 0;

  for (firstTextIndex = 0; firstTextIndex < _player.currentItem.tracks.count; ++firstTextIndex) {

    if ([_player.currentItem.tracks[firstTextIndex].assetTrack hasMediaCharacteristic:AVMediaCharacteristicLegible]) {

      break;

    }

  }

  

  int selectedTrackIndex = -1;

  

  if ([type isEqualToString:@"disabled"]) {

    // Do nothing. We want to ensure option is nil

  } else if ([type isEqualToString:@"language"]) {

    NSString *selectedValue = _selectedTextTrack[@"value"];

    for (int i = 0; i < textTracks.count; ++i) {

      NSDictionary *currentTextTrack = [textTracks objectAtIndex:i];

      if ([selectedValue isEqualToString:currentTextTrack[@"language"]]) {

        selectedTrackIndex = i;

        break;

      }

    }

  } else if ([type isEqualToString:@"title"]) {

    NSString *selectedValue = _selectedTextTrack[@"value"];

    for (int i = 0; i < textTracks.count; ++i) {

      NSDictionary *currentTextTrack = [textTracks objectAtIndex:i];

      if ([selectedValue isEqualToString:currentTextTrack[@"title"]]) {

        selectedTrackIndex = i;

        break;

      }

    }

  } else if ([type isEqualToString:@"index"]) {

    if ([_selectedTextTrack[@"value"] isKindOfClass:[NSNumber class]]) {

      int index = [_selectedTextTrack[@"value"] intValue];

      if (textTracks.count > index) {

        selectedTrackIndex = index;

      }

    }

  }

  

  // user's selected language might not be available, or system defaults have captions enabled

  if (selectedTrackIndex == -1 || [type isEqualToString:@"default"]) {

      CFArrayRef captioningMediaCharacteristics = MACaptionAppearanceCopyPreferredCaptioningMediaCharacteristics(kMACaptionAppearanceDomainUser);

      NSArray *captionSettings = (__bridge NSArray*)captioningMediaCharacteristics;

      if ([captionSettings containsObject: AVMediaCharacteristicTranscribesSpokenDialogForAccessibility]) {

        // iterate through the textTracks to find a matching option, or default to the first object.

        selectedTrackIndex = 0;

        

        NSString * systemLanguage = [[NSLocale preferredLanguages] firstObject];

        for (int i = 0; i < textTracks.count; ++i) {

          NSDictionary *currentTextTrack = [textTracks objectAtIndex:i];

          if ([systemLanguage isEqualToString:currentTextTrack[@"language"]]) {

            selectedTrackIndex = i;

            break;

          }

        }

      }

  }

  

  for (int i = firstTextIndex; i < _player.currentItem.tracks.count; ++i) {

    BOOL isEnabled = i == selectedTrackIndex + firstTextIndex;

    [_player.currentItem.tracks[i] setEnabled:isEnabled];

  }

}



-(void) setStreamingText {

  NSString *type = _selectedTextTrack[@"type"];

  AVMediaSelectionGroup *group = [_player.currentItem.asset

                                  mediaSelectionGroupForMediaCharacteristic:AVMediaCharacteristicLegible];

  AVMediaSelectionOption *option;

  AVMediaSelectionOption *mediaOption;

  

  if ([type isEqualToString:@"disabled"]) {

    // Do nothing. We want to ensure option is nil

  } else if ([type isEqualToString:@"language"] || [type isEqualToString:@"title"]) {

    NSString *value = selectedTextTrack[@"value"];

    NSString *value = _selectedTextTrack[@"value"];

    for (int i = 0; i < group.options.count; ++i) {

      AVMediaSelectionOption *currentOption = [group.options objectAtIndex:i];

      NSString *optionValue;
@@ -688,17 +823,17 @@ static NSString *const timedMetadata = @"timedMetadata"; 
                       objectAtIndex:0];

      }

      if ([value isEqualToString:optionValue]) {

        option = currentOption;

        mediaOption = currentOption;

        break;

      }

    }

    //} else if ([type isEqualToString:@"default"]) {

    //  option = group.defaultOption; */

  } else if ([type isEqualToString:@"index"]) {

    if ([selectedTextTrack[@"value"] isKindOfClass:[NSNumber class]]) {

      int index = [selectedTextTrack[@"value"] intValue];

    if ([_selectedTextTrack[@"value"] isKindOfClass:[NSNumber class]]) {

      int index = [_selectedTextTrack[@"value"] intValue];

      if (group.options.count > index) {

        option = [group.options objectAtIndex:index];

        mediaOption = [group.options objectAtIndex:index];

      }

    }

  } else { // default. invalid type or "system"
@@ -707,11 +842,24 @@ static NSString *const timedMetadata = @"timedMetadata"; 
  }

  

  // If a match isn't found, option will be nil and text tracks will be disabled

  [_player.currentItem selectMediaOption:option inMediaSelectionGroup:group];

  [_player.currentItem selectMediaOption:mediaOption inMediaSelectionGroup:group];

}



- (void)setTextTracks:(NSArray*) textTracks;

{

  _textTracks = textTracks;



  // in case textTracks was set after selectedTextTrack

  if (_selectedTextTrack) [self setSelectedTextTrack:_selectedTextTrack];

}



- (NSArray *)getTextTrackInfo

{

  

  // if sideloaded, textTracks will already be set

  if (_textTracks) return _textTracks;

  

  // if streaming video, we extract the text tracks

  NSMutableArray *textTracks = [[NSMutableArray alloc] init];

  AVMediaSelectionGroup *group = [_player.currentItem.asset

                                  mediaSelectionGroupForMediaCharacteristic:AVMediaCharacteristicLegible];

ios/RCTVideoManager.m

+1 −0
Changes for ios/RCTVideoManager.m: 1 added line, 0 removed lines.
Original line number Diff line number Diff line
@@ -23,6 +23,7 @@ RCT_EXPORT_VIEW_PROPERTY(src, NSDictionary); 
RCT_EXPORT_VIEW_PROPERTY(resizeMode, NSString);

RCT_EXPORT_VIEW_PROPERTY(repeat, BOOL);

RCT_EXPORT_VIEW_PROPERTY(allowsExternalPlayback, BOOL);

RCT_EXPORT_VIEW_PROPERTY(textTracks, NSArray);

RCT_EXPORT_VIEW_PROPERTY(selectedTextTrack, NSDictionary);

RCT_EXPORT_VIEW_PROPERTY(paused, BOOL);

RCT_EXPORT_VIEW_PROPERTY(muted, BOOL);