Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add partial data handling (ie. Progressive JPEG) #478

Draft
wants to merge 1 commit into
base: develop
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions flutter_cache_manager/lib/src/result/interlaced_progress.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
import 'dart:typed_data';

import 'package:flutter_cache_manager/flutter_cache_manager.dart';

class InterlacedProgress extends DownloadProgress {
InterlacedProgress(
super.originalUrl, super.totalSize, super.downloaded, this.data);

final Uint8List data;
}
1 change: 1 addition & 0 deletions flutter_cache_manager/lib/src/result/result.dart
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
export 'download_progress.dart';
export 'file_info.dart';
export 'file_response.dart';
export 'interlaced_progress.dart';
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
import 'dart:convert';
import 'dart:typed_data';

import 'package:flutter_cache_manager/src/web/interlaced/progressive_jpeg_decoder.dart';

class InterlacedData {
final Uint8List data;
InterlacedData(this.data);
}

class InterlacedConverter extends Converter<List<int>, InterlacedData> {
const InterlacedConverter();

@override
InterlacedData convert(List<int> input) =>
InterlacedData(Uint8List.fromList(input));

@override
Sink<Uint8List> startChunkedConversion(Sink<InterlacedData> sink) =>
InterlacedByteConversionSink(sink);
}

class InterlacedByteConversionSink implements ChunkedConversionSink<Uint8List> {
final Sink<InterlacedData> _output;

// Buffer to accumulate chunks
BytesBuilder? _buffer = BytesBuilder();

InterlacedDecoder? _decoder;

InterlacedByteConversionSink(this._output);

@override
void add(List<int> chunk) {
// Ensure buffer is not null (should not happen in normal flow)
final buffer = _buffer;
if (buffer == null) {
throw StateError('Sink has been closed and cannot accept new data.');
}

_decoder ??= resolveDecoder();

if (_decoder == null) {
return _buffer!.add(chunk);
}

final interlacedData = _decoder?.addChunk(chunk);
if (interlacedData != null) {
_output.add(interlacedData);
}
}

@override
void close() {
_buffer?.clear();
_buffer = null;
_decoder = null;
_output.close();
}

InterlacedDecoder? resolveDecoder() {
if (ProgressiveJPEGDecoder.isProgressiveJPEG(_buffer)) {
return ProgressiveJPEGDecoder(_buffer!);
}
return null;
}
}

// Base class for interlaced format decoders
abstract class InterlacedDecoder {
final BytesBuilder buffer;

InterlacedDecoder(this.buffer);

InterlacedData? addChunk(List<int> chunk);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
import 'dart:typed_data';

import 'package:flutter_cache_manager/src/web/interlaced/interlaced_transformer.dart';

// Decoder for progressive JPEG images
class ProgressiveJPEGDecoder extends InterlacedDecoder {
static bool isProgressiveJPEG(BytesBuilder? buffer) {
if (buffer == null) return false;

final data = buffer.toBytes();

if (data.length < 4) return false;

// Check for the SOI (Start of Image)
if (data[0] == 0xFF && data[1] == 0xD8) {
// Check for the first SOF marker
for (int i = 2; i < data.length - 1; i++) {
if (data[i] == 0xFF && data[i + 1] >= 0xC0 && data[i + 1] <= 0xCF) {
return data[i + 1] == 0xC2;
}
}
}

return false;
}

// List of valid offsets
final List<int> _validOffsets = [];

ProgressiveJPEGDecoder(super.buffer);

@override
InterlacedData? addChunk(List<int> chunk) {
// Calculate startOffset before adding the new chunk
int startOffset =
(buffer.length - 1).clamp(0, buffer.length); // Ensure valid bounds

// Add the new chunk to the buffer
buffer.add(chunk);

_updateOffsets(buffer.toBytes(), startOffset);

return _getBestData();
}

InterlacedData? _getBestData() {
// Get the best valid data using the latest valid offset
if (_validOffsets.isNotEmpty) {
final data =
Uint8List.sublistView(buffer.toBytes(), 0, _validOffsets.last);

data[data.length - 1] = 0xd9; // Fix the last byte as EOI
return InterlacedData(data);
}

return null;
}

void _updateOffsets(Uint8List data, int startOffset) {
// Iterate through data starting from the adjusted offset
for (int i = startOffset; i < data.length - 1; i++) {
if (data[i] == 0xFF && _isMarker(data[i + 1])) {
// Add offset of the marker's end
_validOffsets.add(i + 2);
}
}
}

bool _isMarker(int byte) {
// Check for JPEG markers: 0xDA (Start of Scan) or 0xD9 (End of Image)
return byte == 0xDA || byte == 0xD9;
}
}
39 changes: 29 additions & 10 deletions flutter_cache_manager/lib/src/web/web_helper.dart
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import 'package:clock/clock.dart';
import 'package:flutter/foundation.dart';
import 'package:flutter_cache_manager/flutter_cache_manager.dart';
import 'package:flutter_cache_manager/src/cache_store.dart';
import 'package:flutter_cache_manager/src/web/interlaced/interlaced_transformer.dart';
import 'package:flutter_cache_manager/src/web/queue_item.dart';
import 'package:rxdart/rxdart.dart';
import 'package:uuid/uuid.dart';
Expand Down Expand Up @@ -131,11 +132,30 @@ class WebHelper {
var newCacheObject = _setDataFromHeaders(cacheObject, response);
if (statusCodesNewFile.contains(response.statusCode)) {
var savedBytes = 0;
await for (final progress in _saveFile(newCacheObject, response)) {
savedBytes = progress;
yield DownloadProgress(
cacheObject.url, response.contentLength, progress);
final chunkStream =
_saveFile(newCacheObject, response).asBroadcastStream();

final stream = MergeStream([
chunkStream,
chunkStream
.transform<InterlacedData>(const InterlacedConverter())
.distinctUnique(
equals: (a, b) => a.data.length == b.data.length,
hashCode: (e) => e.data.length,
),
]);

await for (final progress in stream) {
if (progress is InterlacedData) {
yield InterlacedProgress(cacheObject.url, response.contentLength,
progress.data.length, progress.data);
} else if (progress is List<int>) {
savedBytes += progress.length;
yield DownloadProgress(
cacheObject.url, response.contentLength, savedBytes);
}
}

newCacheObject = newCacheObject.copyWith(length: savedBytes);
}

Expand Down Expand Up @@ -177,8 +197,9 @@ class WebHelper {
);
}

Stream<int> _saveFile(CacheObject cacheObject, FileServiceResponse response) {
final receivedBytesResultController = StreamController<int>();
Stream<List<int>> _saveFile(
CacheObject cacheObject, FileServiceResponse response) {
final receivedBytesResultController = StreamController<List<int>>();
_saveFileAndPostUpdates(
receivedBytesResultController,
cacheObject,
Expand All @@ -188,17 +209,15 @@ class WebHelper {
}

Future<void> _saveFileAndPostUpdates(
StreamController<int> receivedBytesResultController,
StreamController<List<int>> receivedBytesResultController,
CacheObject cacheObject,
FileServiceResponse response) async {
final file = await _store.fileSystem.createFile(cacheObject.relativePath);

try {
var receivedBytes = 0;
final sink = file.openWrite();
await response.content.map((s) {
receivedBytes += s.length;
receivedBytesResultController.add(receivedBytes);
receivedBytesResultController.add(s);
return s;
}).pipe(sink);
} on Object catch (e, stacktrace) {
Expand Down