Skip to content

Commit

Permalink
Fix: Generate thumbnail by considering the rotation (#82)
Browse files Browse the repository at this point in the history 
* generate thumbnail by considering the rotation

* fix some issues

* rotate image in kotlin layer using matrix on bitmap
  • Loading branch information
@anilbeesetti
anilbeesetti authored Jul 28, 2024
1 parent d70a586 commit 0fb28ac
Show file tree
Hide file tree
Showing 7 changed files with 226 additions and 21 deletions.
154 changes: 150 additions & 4 deletions mediainfo/src/main/cpp/frame_extractor.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,15 @@ extern "C" {
#include <libavcodec/avcodec.h>
#include <libswscale/swscale.h>
#include <libavutil/imgutils.h>
#include <libavutil/display.h>
}

#include <android/bitmap.h>
#include "frame_loader_context.h"
#include "log.h"

bool read_frame(FrameLoaderContext *frameLoaderContext, AVPacket *packet, AVFrame *frame, AVCodecContext *videoCodecContext) {
bool read_frame(FrameLoaderContext *frameLoaderContext, AVPacket *packet, AVFrame *frame,
AVCodecContext *videoCodecContext) {
while (av_read_frame(frameLoaderContext->avFormatContext, packet) >= 0) {
if (packet->stream_index != frameLoaderContext->videoStreamIndex) {
continue;
Expand All @@ -33,7 +35,8 @@ bool read_frame(FrameLoaderContext *frameLoaderContext, AVPacket *packet, AVFram
}


bool frame_extractor_load_frame(JNIEnv *env, int64_t jFrameLoaderContextHandle, int64_t time_millis, jobject jBitmap) {
bool frame_extractor_load_frame(JNIEnv *env, int64_t jFrameLoaderContextHandle, int64_t time_millis,
jobject jBitmap) {
AndroidBitmapInfo bitmapMetricInfo;
AndroidBitmap_getInfo(env, jBitmap, &bitmapMetricInfo);

Expand Down Expand Up @@ -67,7 +70,8 @@ bool frame_extractor_load_frame(JNIEnv *env, int64_t jFrameLoaderContextHandle,
int64_t videoDuration = avVideoStream->duration;
// In some cases the duration is of a video stream is set to Long.MIN_VALUE and we need compute it in another way
if (videoDuration == LONG_LONG_MIN && avVideoStream->time_base.den != 0) {
videoDuration = frameLoaderContext->avFormatContext->duration / avVideoStream->time_base.den;
videoDuration =
frameLoaderContext->avFormatContext->duration / avVideoStream->time_base.den;
}


Expand Down Expand Up @@ -141,6 +145,140 @@ bool frame_extractor_load_frame(JNIEnv *env, int64_t jFrameLoaderContextHandle,
return resultValue;
}

jobject frame_extractor_get_frame(JNIEnv *env, int64_t jFrameLoaderContextHandle, int64_t time_millis) {
auto *frameLoaderContext = frame_loader_context_from_handle(jFrameLoaderContextHandle);
if (!frameLoaderContext || !frameLoaderContext->avFormatContext ||
!frameLoaderContext->parameters) {
return nullptr;
}

auto pixelFormat = static_cast<AVPixelFormat>(frameLoaderContext->parameters->format);
if (pixelFormat == AV_PIX_FMT_NONE) {
return nullptr;
}

AVStream *avVideoStream = frameLoaderContext->avFormatContext->streams[frameLoaderContext->videoStreamIndex];
if (!avVideoStream) {
return nullptr;
}

int srcW = frameLoaderContext->parameters->width;
int srcH = frameLoaderContext->parameters->height;

// Determine bitmap dimensions based on rotation
int bitmapWidth = srcW > 0 ? srcW : 1920;
int bitmapHeight = srcH > 0 ? srcH : 1080;

// Create Java Bitmap
jclass bitmapClass = env->FindClass("android/graphics/Bitmap");
jmethodID createBitmapMethod = env->GetStaticMethodID(bitmapClass, "createBitmap",
"(IILandroid/graphics/Bitmap$Config;)Landroid/graphics/Bitmap;");
jclass bitmapConfigClass = env->FindClass("android/graphics/Bitmap$Config");
jfieldID argb8888FieldID = env->GetStaticFieldID(bitmapConfigClass, "ARGB_8888",
"Landroid/graphics/Bitmap$Config;");
jobject argb8888Obj = env->GetStaticObjectField(bitmapConfigClass, argb8888FieldID);
jobject jBitmap = env->CallStaticObjectMethod(bitmapClass, createBitmapMethod, bitmapWidth,
bitmapHeight, argb8888Obj);

SwsContext *scalingContext = sws_getContext(
srcW, srcH, pixelFormat,
bitmapWidth, bitmapHeight, AV_PIX_FMT_RGBA,
SWS_BICUBIC, nullptr, nullptr, nullptr);

if (!scalingContext) {
return nullptr;
}

int64_t videoDuration = avVideoStream->duration;
if (videoDuration == LONG_LONG_MIN && avVideoStream->time_base.den != 0) {
videoDuration = av_rescale_q(frameLoaderContext->avFormatContext->duration, AV_TIME_BASE_Q,
avVideoStream->time_base);
}

int64_t seekPosition = (time_millis != -1) ?
av_rescale_q(time_millis, AV_TIME_BASE_Q, avVideoStream->time_base) :
videoDuration / 3;

seekPosition = FFMIN(seekPosition, videoDuration);

AVPacket *packet = av_packet_alloc();
AVFrame *frame = av_frame_alloc();
if (!packet || !frame) {
sws_freeContext(scalingContext);
av_packet_free(&packet);
av_frame_free(&frame);
return nullptr;
}

AVCodecContext *videoCodecContext = avcodec_alloc_context3(frameLoaderContext->avVideoCodec);
if (!videoCodecContext ||
avcodec_parameters_to_context(videoCodecContext, frameLoaderContext->parameters) < 0 ||
avcodec_open2(videoCodecContext, frameLoaderContext->avVideoCodec, nullptr) < 0) {
sws_freeContext(scalingContext);
av_packet_free(&packet);
av_frame_free(&frame);
avcodec_free_context(&videoCodecContext);
return nullptr;
}

av_seek_frame(frameLoaderContext->avFormatContext,
frameLoaderContext->videoStreamIndex,
seekPosition,
AVSEEK_FLAG_BACKWARD);

bool resultValue = read_frame(frameLoaderContext, packet, frame, videoCodecContext);

if (!resultValue) {
av_seek_frame(frameLoaderContext->avFormatContext,
frameLoaderContext->videoStreamIndex,
0,
0);
resultValue = read_frame(frameLoaderContext, packet, frame, videoCodecContext);
}

if (resultValue) {
void *bitmapBuffer;
if (AndroidBitmap_lockPixels(env, jBitmap, &bitmapBuffer) < 0) {
resultValue = false;
} else {
AVFrame *frameForDrawing = av_frame_alloc();
if (frameForDrawing) {
av_image_fill_arrays(frameForDrawing->data,
frameForDrawing->linesize,
static_cast<const uint8_t *>(bitmapBuffer),
AV_PIX_FMT_RGBA,
bitmapWidth,
bitmapHeight,
1);
sws_scale(scalingContext,
frame->data,
frame->linesize,
0,
frame->height,
frameForDrawing->data,
frameForDrawing->linesize);

av_frame_free(&frameForDrawing);
}
AndroidBitmap_unlockPixels(env, jBitmap);
}
}

av_packet_free(&packet);
av_frame_free(&frame);
avcodec_free_context(&videoCodecContext);
sws_freeContext(scalingContext);

if (resultValue) {
return jBitmap;
} else {
env->DeleteLocalRef(jBitmap);
return nullptr;
}

}


extern "C"
JNIEXPORT void JNICALL
Java_io_github_anilbeesetti_nextlib_mediainfo_FrameLoader_nativeRelease(JNIEnv *env, jclass clazz,
Expand All @@ -153,6 +291,14 @@ Java_io_github_anilbeesetti_nextlib_mediainfo_FrameLoader_nativeLoadFrame(JNIEnv
jlong jFrameLoaderContextHandle,
jlong time_millis,
jobject jBitmap) {
bool successfullyLoaded = frame_extractor_load_frame(env, jFrameLoaderContextHandle, time_millis, jBitmap);
bool successfullyLoaded = frame_extractor_load_frame(env, jFrameLoaderContextHandle,
time_millis, jBitmap);
return static_cast<jboolean>(successfullyLoaded);
}
extern "C"
JNIEXPORT jobject JNICALL
Java_io_github_anilbeesetti_nextlib_mediainfo_FrameLoader_nativeGetFrame(JNIEnv *env, jclass clazz,
jlong jFrameLoaderContextHandle,
jlong time_millis) {
return frame_extractor_get_frame(env, jFrameLoaderContextHandle, time_millis);
}
56 changes: 43 additions & 13 deletions mediainfo/src/main/cpp/mediainfo.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,9 @@
#include "frame_loader_context.h"

extern "C" {
#include "libavformat/avformat.h"
#include "libavcodec/codec_desc.h"
#include <libavformat/avformat.h>
#include <libavcodec/codec_desc.h>
#include <libavutil/display.h>
}

static char *get_string(AVDictionary *metadata, const char *key) {
Expand Down Expand Up @@ -43,7 +44,8 @@ void onMediaInfoFound(JNIEnv *env, jobject jMediaInfoBuilder, AVFormatContext *a
duration_ms);
}

void onVideoStreamFound(JNIEnv *env, jobject jMediaInfoBuilder, AVFormatContext *avFormatContext, int index) {
void onVideoStreamFound(JNIEnv *env, jobject jMediaInfoBuilder, AVFormatContext *avFormatContext,
int index) {
AVStream *stream = avFormatContext->streams[index];
AVCodecParameters *parameters = stream->codecpar;

Expand All @@ -61,9 +63,12 @@ void onVideoStreamFound(JNIEnv *env, jobject jMediaInfoBuilder, AVFormatContext
}


AVRational guessedFrameRate = av_guess_frame_rate(avFormatContext, avFormatContext->streams[index], nullptr);
AVRational guessedFrameRate = av_guess_frame_rate(avFormatContext,
avFormatContext->streams[index],
nullptr);

double resultFrameRate = guessedFrameRate.den == 0 ? 0.0 : guessedFrameRate.num / (double) guessedFrameRate.den;
double resultFrameRate =
guessedFrameRate.den == 0 ? 0.0 : guessedFrameRate.num / (double) guessedFrameRate.den;

jstring jTitle = env->NewStringUTF(get_title(stream->metadata));
jstring jCodecName;
Expand All @@ -74,6 +79,22 @@ void onVideoStreamFound(JNIEnv *env, jobject jMediaInfoBuilder, AVFormatContext
}
jstring jLanguage = env->NewStringUTF(get_language(stream->metadata));

int rotation = 0;
AVDictionaryEntry *rotateTag = av_dict_get(stream->metadata, "rotate", nullptr, 0);
if (rotateTag && *rotateTag->value) {
rotation = atoi(rotateTag->value);
rotation %= 360;
if (rotation < 0) rotation += 360;
}
uint8_t *displaymatrix = av_stream_get_side_data(stream,
AV_PKT_DATA_DISPLAYMATRIX,
nullptr);
if (displaymatrix) {
double theta = av_display_rotation_get((int32_t *) displaymatrix);
rotation = (int) (-theta) % 360;
if (rotation < 0) rotation += 360;
}

utils_call_instance_method_void(env,
jMediaInfoBuilder,
fields.MediaInfoBuilder.onVideoStreamFoundID,
Expand All @@ -86,10 +107,12 @@ void onVideoStreamFound(JNIEnv *env, jobject jMediaInfoBuilder, AVFormatContext
resultFrameRate,
parameters->width,
parameters->height,
rotation,
frameLoaderContextHandle);
}

void onAudioStreamFound(JNIEnv *env, jobject jMediaInfoBuilder, AVFormatContext *avFormatContext, int index) {
void onAudioStreamFound(JNIEnv *env, jobject jMediaInfoBuilder, AVFormatContext *avFormatContext,
int index) {
AVStream *stream = avFormatContext->streams[index];
AVCodecParameters *parameters = stream->codecpar;

Expand All @@ -98,7 +121,8 @@ void onAudioStreamFound(JNIEnv *env, jobject jMediaInfoBuilder, AVFormatContext
auto avSampleFormat = static_cast<AVSampleFormat>(parameters->format);
auto jSampleFormat = env->NewStringUTF(av_get_sample_fmt_name(avSampleFormat));
char chLayoutDescription[128];
av_channel_layout_describe(&parameters->ch_layout, chLayoutDescription, sizeof(chLayoutDescription));
av_channel_layout_describe(&parameters->ch_layout, chLayoutDescription,
sizeof(chLayoutDescription));

jstring jTitle = env->NewStringUTF(get_title(stream->metadata));
jstring jCodecName;
Expand All @@ -125,7 +149,8 @@ void onAudioStreamFound(JNIEnv *env, jobject jMediaInfoBuilder, AVFormatContext
jChannelLayout);
}

void onSubtitleStreamFound(JNIEnv *env, jobject jMediaInfoBuilder, AVFormatContext *avFormatContext, int index) {
void onSubtitleStreamFound(JNIEnv *env, jobject jMediaInfoBuilder, AVFormatContext *avFormatContext,
int index) {
AVStream *stream = avFormatContext->streams[index];
AVCodecParameters *parameters = stream->codecpar;

Expand All @@ -150,11 +175,12 @@ void onSubtitleStreamFound(JNIEnv *env, jobject jMediaInfoBuilder, AVFormatConte
stream->disposition);
}

void onChapterFound(JNIEnv *env, jobject jMediaInfoBuilder, AVFormatContext *avFormatContext, int index) {
void onChapterFound(JNIEnv *env, jobject jMediaInfoBuilder, AVFormatContext *avFormatContext,
int index) {
AVChapter *chapter = avFormatContext->chapters[index];

jstring jTitle = env->NewStringUTF(get_title(chapter->metadata));
double time_base = av_q2d(chapter->time_base);
double time_base = av_q2d(chapter->time_base);
long start_ms = (long) (chapter->start * time_base * 1000.0);
long end_ms = (long) (chapter->end * time_base * 1000.0);

Expand Down Expand Up @@ -189,7 +215,7 @@ void media_info_build(JNIEnv *env, jobject jMediaInfoBuilder, const char *uri) {
AVMediaType type = parameters->codec_type;
switch (type) {
case AVMEDIA_TYPE_VIDEO:
onVideoStreamFound(env, jMediaInfoBuilder, avFormatContext, pos);
onVideoStreamFound(env, jMediaInfoBuilder, avFormatContext, pos);
break;
case AVMEDIA_TYPE_AUDIO:
onAudioStreamFound(env, jMediaInfoBuilder, avFormatContext, pos);
Expand All @@ -207,7 +233,9 @@ void media_info_build(JNIEnv *env, jobject jMediaInfoBuilder, const char *uri) {

extern "C"
JNIEXPORT void JNICALL
Java_io_github_anilbeesetti_nextlib_mediainfo_MediaInfoBuilder_nativeCreateFromFD(JNIEnv *env, jobject thiz, jint file_descriptor) {
Java_io_github_anilbeesetti_nextlib_mediainfo_MediaInfoBuilder_nativeCreateFromFD(JNIEnv *env,
jobject thiz,
jint file_descriptor) {
char pipe[32];
sprintf(pipe, "pipe:%d", file_descriptor);

Expand All @@ -216,7 +244,9 @@ Java_io_github_anilbeesetti_nextlib_mediainfo_MediaInfoBuilder_nativeCreateFromF

extern "C"
JNIEXPORT void JNICALL
Java_io_github_anilbeesetti_nextlib_mediainfo_MediaInfoBuilder_nativeCreateFromPath(JNIEnv *env, jobject thiz, jstring jFilePath) {
Java_io_github_anilbeesetti_nextlib_mediainfo_MediaInfoBuilder_nativeCreateFromPath(JNIEnv *env,
jobject thiz,
jstring jFilePath) {
const char *cFilePath = env->GetStringUTFChars(jFilePath, nullptr);

media_info_build(env, thiz, cFilePath);
Expand Down
2 changes: 1 addition & 1 deletion mediainfo/src/main/cpp/utils.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ int utils_fields_init(JavaVM *vm) {
GET_ID(GetMethodID,
fields.MediaInfoBuilder.onVideoStreamFoundID,
fields.MediaInfoBuilder.clazz,
"onVideoStreamFound", "(ILjava/lang/String;Ljava/lang/String;Ljava/lang/String;IJDIIJ)V"
"onVideoStreamFound", "(ILjava/lang/String;Ljava/lang/String;Ljava/lang/String;IJDIIIJ)V"
);

GET_ID(GetMethodID,
Expand Down
8 changes: 8 additions & 0 deletions mediainfo/src/main/java/io/github/anilbeesetti/nextlib/mediainfo/FrameLoader.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,11 @@ class FrameLoader internal constructor(private var frameLoaderContextHandle: Lon
return nativeLoadFrame(frameLoaderContextHandle, durationMillis, bitmap)
}

fun getFrame(durationMillis: Long): Bitmap? {
require(frameLoaderContextHandle != -1L)
return nativeGetFrame(frameLoaderContextHandle, durationMillis)
}

fun release() {
nativeRelease(frameLoaderContextHandle)
frameLoaderContextHandle = -1
Expand All @@ -20,5 +25,8 @@ class FrameLoader internal constructor(private var frameLoaderContextHandle: Lon

@JvmStatic
private external fun nativeLoadFrame(handle: Long, durationMillis: Long, bitmap: Bitmap): Boolean

@JvmStatic
private external fun nativeGetFrame(handle: Long, durationMillis: Long): Bitmap?
}
}
20 changes: 19 additions & 1 deletion mediainfo/src/main/java/io/github/anilbeesetti/nextlib/mediainfo/MediaInfo.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,29 @@ data class MediaInfo(
if (videoStream == null) return null
val bitmap = Bitmap.createBitmap(videoStream.frameWidth.takeIf { it > 0 } ?: 1920, videoStream.frameHeight.takeIf { it > 0 } ?: 1080, Bitmap.Config.ARGB_8888)
val result = frameLoader?.loadFrameInto(bitmap, durationMillis)
return if (result == true) bitmap else null
return if (result == true) bitmap.rotate(videoStream.rotation) else null
}

/**
* Retrieves a video frame as a Bitmap at a specific duration in milliseconds from the video stream.
*
* @param durationMillis The timestamp in milliseconds at which to retrieve the video frame.
* If set to -1, the frame will be retrieved at one-third of the video's duration.
* @return A Bitmap containing the video frame if retrieval is successful, or null if an error occurs.
*/
fun getFrameAt(durationMillis: Long = -1): Bitmap? {
if (videoStream == null) return null
return frameLoader?.getFrame(durationMillis)?.rotate(videoStream.rotation)
}

fun release() {
frameLoader?.release()
frameLoader = null
}
}

private fun Bitmap.rotate(degrees: Int): Bitmap {
val matrix = android.graphics.Matrix()
matrix.postRotate(degrees.toFloat())
return Bitmap.createBitmap(this, 0, 0, width, height, matrix, true)
}
Loading

0 comments on commit 0fb28ac

Please sign in to comment.