android: add subtitle supports (bilibili#2450)

* android: add subtitle supports * supplements of subtitle meta * add post_event2 for posting an object to java * fix memleak, using malloced area to store obj data to isolate life cycle
hyqgod · Dec 30, 2016 · 24b6014 · 24b6014
1 parent dfac848
commit 24b6014
Show file tree

Hide file tree

Showing 20 changed files with 358 additions and 64 deletions.
diff --git a/...jkplayer-example/src/main/java/tv/danmaku/ijk/media/example/fragments/TracksFragment.java b/...jkplayer-example/src/main/java/tv/danmaku/ijk/media/example/fragments/TracksFragment.java
@@ -67,10 +67,13 @@ public void onActivityCreated(@Nullable Bundle savedInstanceState) {
 
             int selectedVideoTrack = trackHolder.getSelectedTrack(ITrackInfo.MEDIA_TRACK_TYPE_VIDEO);
             int selectedAudioTrack = trackHolder.getSelectedTrack(ITrackInfo.MEDIA_TRACK_TYPE_AUDIO);
+            int selectedSubtitleTrack = trackHolder.getSelectedTrack(ITrackInfo.MEDIA_TRACK_TYPE_TIMEDTEXT);
             if (selectedVideoTrack >= 0)
                 mTrackListView.setItemChecked(selectedVideoTrack, true);
             if (selectedAudioTrack >= 0)
                 mTrackListView.setItemChecked(selectedAudioTrack, true);
+            if (selectedSubtitleTrack >= 0)
+                mTrackListView.setItemChecked(selectedSubtitleTrack, true);
 
             mTrackListView.setOnItemClickListener(new AdapterView.OnItemClickListener() {
                 @Override

diff --git a/...kplayer-example/src/main/java/tv/danmaku/ijk/media/example/widget/media/IjkVideoView.java b/...kplayer-example/src/main/java/tv/danmaku/ijk/media/example/widget/media/IjkVideoView.java
@@ -36,18 +36,22 @@
 import android.widget.FrameLayout;
 import android.widget.MediaController;
 import android.widget.TableLayout;
+import android.widget.TextView;
 
 import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import tv.danmaku.ijk.media.exo.IjkExoMediaPlayer;
 import tv.danmaku.ijk.media.player.AndroidMediaPlayer;
 import tv.danmaku.ijk.media.player.IMediaPlayer;
 import tv.danmaku.ijk.media.player.IjkMediaPlayer;
+import tv.danmaku.ijk.media.player.IjkTimedText;
 import tv.danmaku.ijk.media.player.TextureMediaPlayer;
 import tv.danmaku.ijk.media.player.misc.IMediaDataSource;
 import tv.danmaku.ijk.media.player.misc.IMediaFormat;
@@ -122,6 +126,8 @@ public class IjkVideoView extends FrameLayout implements MediaController.MediaPl
     private long mSeekStartTime = 0;
     private long mSeekEndTime = 0;
 
+    private TextView subtitleDisplay;
+
     public IjkVideoView(Context context) {
         super(context);
         initVideoView(context);
@@ -165,6 +171,15 @@ private void initVideoView(Context context) {
         // REMOVED: mPendingSubtitleTracks = new Vector<Pair<InputStream, MediaFormat>>();
         mCurrentState = STATE_IDLE;
         mTargetState = STATE_IDLE;
+
+        subtitleDisplay = new TextView(context);
+        subtitleDisplay.setTextSize(24);
+        subtitleDisplay.setGravity(Gravity.CENTER);
+        FrameLayout.LayoutParams layoutParams_txt = new FrameLayout.LayoutParams(
+                FrameLayout.LayoutParams.MATCH_PARENT,
+                FrameLayout.LayoutParams.WRAP_CONTENT,
+                Gravity.BOTTOM);
+        addView(subtitleDisplay, layoutParams_txt);
     }
 
     public void setRenderView(IRenderView renderView) {
@@ -314,6 +329,7 @@ private void openVideo() {
             mMediaPlayer.setOnInfoListener(mInfoListener);
             mMediaPlayer.setOnBufferingUpdateListener(mBufferingUpdateListener);
             mMediaPlayer.setOnSeekCompleteListener(mSeekCompleteListener);
+            mMediaPlayer.setOnTimedTextListener(mOnTimedTextListener);
             mCurrentBufferPercentage = 0;
             String scheme = mUri.getScheme();
             if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M &&
@@ -580,6 +596,15 @@ public void onSeekComplete(IMediaPlayer mp) {
         }
     };
 
+    private IMediaPlayer.OnTimedTextListener mOnTimedTextListener = new IMediaPlayer.OnTimedTextListener() {
+        @Override
+        public void onTimedText(IMediaPlayer mp, IjkTimedText text) {
+            if (text != null) {
+                subtitleDisplay.setText(text.getText());
+            }
+        }
+    };
+
     /**
      * Register a callback to be invoked when the media file
      * is loaded and ready to go.
@@ -1095,6 +1120,7 @@ public void showMediaInfo() {
 
         int selectedVideoTrack = MediaPlayerCompat.getSelectedTrack(mMediaPlayer, ITrackInfo.MEDIA_TRACK_TYPE_VIDEO);
         int selectedAudioTrack = MediaPlayerCompat.getSelectedTrack(mMediaPlayer, ITrackInfo.MEDIA_TRACK_TYPE_AUDIO);
+        int selectedSubtitleTrack = MediaPlayerCompat.getSelectedTrack(mMediaPlayer, ITrackInfo.MEDIA_TRACK_TYPE_TIMEDTEXT);
 
         TableLayoutBinder builder = new TableLayoutBinder(getContext());
         builder.appendSection(R.string.mi_player);
@@ -1114,6 +1140,8 @@ public void showMediaInfo() {
                     builder.appendSection(getContext().getString(R.string.mi_stream_fmt1, index) + " " + getContext().getString(R.string.mi__selected_video_track));
                 } else if (index == selectedAudioTrack) {
                     builder.appendSection(getContext().getString(R.string.mi_stream_fmt1, index) + " " + getContext().getString(R.string.mi__selected_audio_track));
+                } else if (index == selectedSubtitleTrack) {
+                    builder.appendSection(getContext().getString(R.string.mi_stream_fmt1, index) + " " + getContext().getString(R.string.mi__selected_subtitle_track));
                 } else {
                     builder.appendSection(getContext().getString(R.string.mi_stream_fmt1, index));
                 }

diff --git a/android/ijkplayer/ijkplayer-example/src/main/res/values/strings.xml b/android/ijkplayer/ijkplayer-example/src/main/res/values/strings.xml
@@ -41,6 +41,7 @@
     <string name="mi_channels">Channels</string>
     <string name="mi__selected_video_track">*</string>
     <string name="mi__selected_audio_track">*</string>
+    <string name="mi__selected_subtitle_track">*</string>
 
     <string name="TrackType_video">Video</string>
     <string name="TrackType_audio">Audio</string>

diff --git a/...kplayer/ijkplayer-java/src/main/java/tv/danmaku/ijk/media/player/AbstractMediaPlayer.java b/...kplayer/ijkplayer-java/src/main/java/tv/danmaku/ijk/media/player/AbstractMediaPlayer.java
@@ -27,6 +27,7 @@ public abstract class AbstractMediaPlayer implements IMediaPlayer {
     private OnVideoSizeChangedListener mOnVideoSizeChangedListener;
     private OnErrorListener mOnErrorListener;
     private OnInfoListener mOnInfoListener;
+    private OnTimedTextListener mOnTimedTextListener;
 
     public final void setOnPreparedListener(OnPreparedListener listener) {
         mOnPreparedListener = listener;
@@ -58,6 +59,10 @@ public final void setOnInfoListener(OnInfoListener listener) {
         mOnInfoListener = listener;
     }
 
+    public final void setOnTimedTextListener(OnTimedTextListener listener) {
+        mOnTimedTextListener = listener;
+    }
+
     public void resetListeners() {
         mOnPreparedListener = null;
         mOnBufferingUpdateListener = null;
@@ -66,6 +71,7 @@ public void resetListeners() {
         mOnVideoSizeChangedListener = null;
         mOnErrorListener = null;
         mOnInfoListener = null;
+        mOnTimedTextListener = null;
     }
 
     protected final void notifyOnPrepared() {
@@ -103,6 +109,11 @@ protected final boolean notifyOnInfo(int what, int extra) {
         return mOnInfoListener != null && mOnInfoListener.onInfo(this, what, extra);
     }
 
+    protected final void notifyOnTimedText(IjkTimedText text) {
+        if (mOnTimedTextListener != null)
+            mOnTimedTextListener.onTimedText(this, text);
+    }
+
     public void setDataSource(IMediaDataSource mediaDataSource) {
         throw new UnsupportedOperationException();
     }

diff --git a/...jkplayer/ijkplayer-java/src/main/java/tv/danmaku/ijk/media/player/AndroidMediaPlayer.java b/...jkplayer/ijkplayer-java/src/main/java/tv/danmaku/ijk/media/player/AndroidMediaPlayer.java
@@ -22,6 +22,7 @@
 import android.media.AudioManager;
 import android.media.MediaDataSource;
 import android.media.MediaPlayer;
+import android.media.TimedText;
 import android.net.Uri;
 import android.os.Build;
 import android.text.TextUtils;
@@ -342,14 +343,16 @@ private void attachInternalListeners() {
                 .setOnVideoSizeChangedListener(mInternalListenerAdapter);
         mInternalMediaPlayer.setOnErrorListener(mInternalListenerAdapter);
         mInternalMediaPlayer.setOnInfoListener(mInternalListenerAdapter);
+        mInternalMediaPlayer.setOnTimedTextListener(mInternalListenerAdapter);
     }
 
     private class AndroidMediaPlayerListenerHolder implements
             MediaPlayer.OnPreparedListener, MediaPlayer.OnCompletionListener,
             MediaPlayer.OnBufferingUpdateListener,
             MediaPlayer.OnSeekCompleteListener,
             MediaPlayer.OnVideoSizeChangedListener,
-            MediaPlayer.OnErrorListener, MediaPlayer.OnInfoListener {
+            MediaPlayer.OnErrorListener, MediaPlayer.OnInfoListener,
+            MediaPlayer.OnTimedTextListener {
         public final WeakReference<AndroidMediaPlayer> mWeakMediaPlayer;
 
         public AndroidMediaPlayerListenerHolder(AndroidMediaPlayer mp) {
@@ -414,5 +417,15 @@ public void onPrepared(MediaPlayer mp) {
 
             notifyOnPrepared();
         }
+
+        @Override
+        public void onTimedText(MediaPlayer mp, TimedText text) {
+            AndroidMediaPlayer self = mWeakMediaPlayer.get();
+            if (self == null)
+                return;
+
+            IjkTimedText ijkText = new IjkTimedText(text.getBounds(), text.getText());
+            notifyOnTimedText(ijkText);
+        }
     }
 }
diff --git a/android/ijkplayer/ijkplayer-java/src/main/java/tv/danmaku/ijk/media/player/IMediaPlayer.java b/android/ijkplayer/ijkplayer-java/src/main/java/tv/danmaku/ijk/media/player/IMediaPlayer.java
@@ -132,6 +132,8 @@ void setOnVideoSizeChangedListener(
 
     void setOnInfoListener(OnInfoListener listener);
 
+    void setOnTimedTextListener(OnTimedTextListener listener);
+
     /*--------------------
      * Listeners
      */
@@ -164,6 +166,10 @@ interface OnInfoListener {
         boolean onInfo(IMediaPlayer mp, int what, int extra);
     }
 
+    interface OnTimedTextListener {
+        void onTimedText(IMediaPlayer mp, IjkTimedText text);
+    }
+
     /*--------------------
      * Optional
      */

diff --git a/android/ijkplayer/ijkplayer-java/src/main/java/tv/danmaku/ijk/media/player/IjkMediaMeta.java b/android/ijkplayer/ijkplayer-java/src/main/java/tv/danmaku/ijk/media/player/IjkMediaMeta.java
@@ -15,11 +15,13 @@ public class IjkMediaMeta {
     public static final String IJKM_KEY_BITRATE = "bitrate";
     public static final String IJKM_KEY_VIDEO_STREAM = "video";
     public static final String IJKM_KEY_AUDIO_STREAM = "audio";
+    public static final String IJKM_KEY_TIMEDTEXT_STREAM = "timedtext";
 
     // stream meta
     public static final String IJKM_KEY_TYPE = "type";
     public static final String IJKM_VAL_TYPE__VIDEO = "video";
     public static final String IJKM_VAL_TYPE__AUDIO = "audio";
+    public static final String IJKM_VAL_TYPE__TIMEDTEXT = "timedtext";
     public static final String IJKM_VAL_TYPE__UNKNOWN = "unknown";
     public static final String IJKM_KEY_LANGUAGE = "language";
 
@@ -202,6 +204,7 @@ public static IjkMediaMeta parse(Bundle mediaMeta) {
 
         int videoStreamIndex = meta.getInt(IJKM_KEY_VIDEO_STREAM, -1);
         int audioStreamIndex = meta.getInt(IJKM_KEY_AUDIO_STREAM, -1);
+        int subtitleStreamIndex = meta.getInt(IJKM_KEY_TIMEDTEXT_STREAM, -1);
 
         ArrayList<Bundle> streams = meta
                 .getParcelableArrayList(IJKM_KEY_STREAMS);

diff --git a/...id/ijkplayer/ijkplayer-java/src/main/java/tv/danmaku/ijk/media/player/IjkMediaPlayer.java b/...id/ijkplayer/ijkplayer-java/src/main/java/tv/danmaku/ijk/media/player/IjkMediaPlayer.java
@@ -23,6 +23,7 @@
 import android.content.Context;
 import android.content.res.AssetFileDescriptor;
 import android.graphics.SurfaceTexture;
+import android.graphics.Rect;
 import android.media.MediaCodecInfo;
 import android.media.MediaCodecList;
 import android.media.RingtoneManager;
@@ -108,6 +109,7 @@ public final class IjkMediaPlayer extends AbstractMediaPlayer {
 
     public static final int FFP_PROP_INT64_SELECTED_VIDEO_STREAM            = 20001;
     public static final int FFP_PROP_INT64_SELECTED_AUDIO_STREAM            = 20002;
+    public static final int FFP_PROP_INT64_SELECTED_TIMEDTEXT_STREAM        = 20011;
 
     public static final int FFP_PROP_INT64_VIDEO_DECODER                    = 20003;
     public static final int FFP_PROP_INT64_AUDIO_DECODER                    = 20004;
@@ -584,6 +586,8 @@ public IjkTrackInfo[] getTrackInfo() {
                 trackInfo.setTrackType(ITrackInfo.MEDIA_TRACK_TYPE_VIDEO);
             } else if (streamMeta.mType.equalsIgnoreCase(IjkMediaMeta.IJKM_VAL_TYPE__AUDIO)) {
                 trackInfo.setTrackType(ITrackInfo.MEDIA_TRACK_TYPE_AUDIO);
+            } else if (streamMeta.mType.equalsIgnoreCase(IjkMediaMeta.IJKM_VAL_TYPE__TIMEDTEXT)) {
+                trackInfo.setTrackType(ITrackInfo.MEDIA_TRACK_TYPE_TIMEDTEXT);
             }
             trackInfos.add(trackInfo);
         }
@@ -598,6 +602,8 @@ public int getSelectedTrack(int trackType) {
                 return (int)_getPropertyLong(FFP_PROP_INT64_SELECTED_VIDEO_STREAM, -1);
             case ITrackInfo.MEDIA_TRACK_TYPE_AUDIO:
                 return (int)_getPropertyLong(FFP_PROP_INT64_SELECTED_AUDIO_STREAM, -1);
+            case ITrackInfo.MEDIA_TRACK_TYPE_TIMEDTEXT:
+                return (int)_getPropertyLong(FFP_PROP_INT64_SELECTED_TIMEDTEXT_STREAM, -1);
             default:
                 return -1;
         }
@@ -966,9 +972,13 @@ public void handleMessage(Message msg) {
                 // No real default action so far.
                 return;
             case MEDIA_TIMED_TEXT:
-                // do nothing
-                break;
-
+                if (msg.obj == null) {
+                    player.notifyOnTimedText(null);
+                } else {
+                    IjkTimedText text = new IjkTimedText(new Rect(0, 0, 1, 1), (String)msg.obj);
+                    player.notifyOnTimedText(text);
+                }
+                return;
             case MEDIA_NOP: // interface test message - ignore
                 break;
 

diff --git a/android/ijkplayer/ijkplayer-java/src/main/java/tv/danmaku/ijk/media/player/IjkTimedText.java b/android/ijkplayer/ijkplayer-java/src/main/java/tv/danmaku/ijk/media/player/IjkTimedText.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright (C) 2016 Zheng Yuan <[email protected]>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package tv.danmaku.ijk.media.player;
+
+import android.graphics.Rect;
+import java.lang.String;
+
+public final class IjkTimedText {
+
+    private Rect mTextBounds = null;
+    private String mTextChars = null;
+
+    public IjkTimedText(Rect bounds, String text) {
+        mTextBounds = bounds;
+        mTextChars = text;
+    }
+
+    public Rect getBounds() {
+        return mTextBounds;
+    }
+
+    public String getText() {
+        return mTextChars;
+    }
+}
diff --git a/.../ijkplayer/ijkplayer-java/src/main/java/tv/danmaku/ijk/media/player/MediaPlayerProxy.java b/.../ijkplayer/ijkplayer-java/src/main/java/tv/danmaku/ijk/media/player/MediaPlayerProxy.java
@@ -281,6 +281,21 @@ public boolean onInfo(IMediaPlayer mp, int what, int extra) {
         }
     }
 
+    @Override
+    public void setOnTimedTextListener(OnTimedTextListener listener) {
+        if (listener != null) {
+            final OnTimedTextListener finalListener = listener;
+            mBackEndMediaPlayer.setOnTimedTextListener(new OnTimedTextListener() {
+                @Override
+                public void onTimedText(IMediaPlayer mp, IjkTimedText text) {
+                    finalListener.onTimedText(MediaPlayerProxy.this, text);
+                }
+            });
+        } else {
+            mBackEndMediaPlayer.setOnTimedTextListener(null);
+        }
+    }
+
     @Override
     public void setAudioStreamType(int streamtype) {
         mBackEndMediaPlayer.setAudioStreamType(streamtype);

diff --git a/...ijkplayer/ijkplayer-java/src/main/java/tv/danmaku/ijk/media/player/misc/IjkTrackInfo.java b/...ijkplayer/ijkplayer-java/src/main/java/tv/danmaku/ijk/media/player/misc/IjkTrackInfo.java
@@ -83,6 +83,8 @@ public String getInfoInline() {
                 break;
             case MEDIA_TRACK_TYPE_TIMEDTEXT:
                 out.append("TIMEDTEXT");
+                out.append(", ");
+                out.append(mStreamMeta.mLanguage);
                 break;
             case MEDIA_TRACK_TYPE_SUBTITLE:
                 out.append("SUBTITLE");

diff --git a/ijkmedia/ijkplayer/android/ijkplayer_jni.c b/ijkmedia/ijkplayer/android/ijkplayer_jni.c
@@ -622,6 +622,7 @@ IjkMediaPlayer_getMediaMeta(JNIEnv *env, jobject thiz)
 
     fillMetaInternal(env, jlocal_bundle, meta, IJKM_KEY_VIDEO_STREAM, "-1");
     fillMetaInternal(env, jlocal_bundle, meta, IJKM_KEY_AUDIO_STREAM, "-1");
+    fillMetaInternal(env, jlocal_bundle, meta, IJKM_KEY_TIMEDTEXT_STREAM, "-1");
 
     jarray_list = J4AC_ArrayList__ArrayList(env);
     if (J4A_ExceptionCheck__throwAny(env)) {
@@ -831,6 +832,13 @@ inline static void post_event(JNIEnv *env, jobject weak_this, int what, int arg1
     // MPTRACE("post_event()=void");
 }
 
+inline static void post_event2(JNIEnv *env, jobject weak_this, int what, int arg1, int arg2, jobject obj)
+{
+    // MPTRACE("post_event2(%p, %p, %d, %d, %d, %p)", (void*)env, (void*) weak_this, what, arg1, arg2, (void*)obj);
+    J4AC_IjkMediaPlayer__postEventFromNative(env, weak_this, what, arg1, arg2, obj);
+    // MPTRACE("post_event2()=void");
+}
+
 static void message_loop_n(JNIEnv *env, IjkMediaPlayer *mp)
 {
     jobject weak_thiz = (jobject) ijkmp_get_weak_thiz(mp);
@@ -905,10 +913,21 @@ static void message_loop_n(JNIEnv *env, IjkMediaPlayer *mp)
             break;
         case FFP_MSG_PLAYBACK_STATE_CHANGED:
             break;
+        case FFP_MSG_TIMED_TEXT:
+            if (msg.obj) {
+                jstring text = (*env)->NewStringUTF(env, (char *)msg.obj);
+                post_event2(env, weak_thiz, MEDIA_TIMED_TEXT, 0, 0, text);
+                J4A_DeleteLocalRef__p(env, &text);
+            }
+            else {
+                post_event2(env, weak_thiz, MEDIA_TIMED_TEXT, 0, 0, NULL);
+            }
+            break;
         default:
             ALOGE("unknown FFP_MSG_xxx(%d)\n", msg.what);
             break;
         }
+        msg_free_res(&msg);
     }
 
 LABEL_RETURN: