feat: improve BinaryWriter API (isar#986)

- extend default buffer size
- allow HiveCipher to asynchronously perform crypto
- implement HiveAesThreadedCipher for Hive Flutter
- fix missing file in .gitignore

In theory, no public API should be made incompatible here as HiveCipher
was simply converted from `T Function()` to `FutureOr<T> Function()`.

The idea behind the asynchronous operation is

a) multithreading using e.g. the `compute()` function in Flutter or
b) platform-native implementations making use of hardware-accelerated
cryptographic implementations.

Signed-off-by: TheOneWithTheBraid <[email protected]>

.github/workflows/test.yml

@@ -36,7 +36,7 @@ jobs:
  channel: ${{ matrix.flutter-channel }}
  - name: Override dependency version
  run: |
- echo -e "\ndependency_overrides:\n win32: 2.6.1" >> pubspec.yaml
+ echo -e "\ndependency_overrides:\n win32: 2.6.1\n hive:\n path: ../hive" >> pubspec.yaml
  working-directory: hive_flutter
  - name: Install dependencies
  run: flutter pub get

hive/README.md

@@ -164,6 +164,35 @@ class SettingsPage extends StatelessWidget {
 
 Boxes are cached and therefore fast enough to be used directly in the `build()` method of Flutter widgets.
 
+### Native AES crypto implementation
+
+When using Flutter, Hive supports native encryption using [package:cryptography](https://pub.dev/packages/cryptography)
+and [package:cryptography_flutter](https://pub.dev/packages/cryptography_flutter).
+
+Native AES implementations tremendously speed up operations on encrypted Boxes.
+
+Please follow these steps:
+
+1. add dependency to pubspec.yaml
+
+```yaml
+dependencies:
+ cryptography_flutter: ^2.0.2
+```
+
+2. enable native implementations
+
+```dart
+import 'package:cryptography_flutter/cryptography_flutter.dart';
+
+void main() {
+ // Enable Flutter cryptography
+ FlutterCryptography.enable();
+
+ // ....
+}
+```
+
 ## Benchmark
 
 | 1000 read iterations | 1000 write iterations |

hive/lib/src/backend/js/native/storage_backend_js.dart

@@ -41,7 +41,7 @@ class StorageBackendJs extends StorageBackend {
 
  /// Not part of public API
  @visibleForTesting
- dynamic encodeValue(Frame frame) {
+ Future<dynamic> encodeValue(Frame frame) async {
  var value = frame.value;
  if (_cipher == null) {
  if (value == null) {
@@ -66,7 +66,7 @@ class StorageBackendJs extends StorageBackend {
  if (_cipher == null) {
  frameWriter.write(value);
  } else {
- frameWriter.writeEncrypted(value, _cipher!);
+ await frameWriter.writeEncrypted(value, _cipher!);
  }
 
  var bytes = frameWriter.toBytes();
@@ -76,7 +76,7 @@ class StorageBackendJs extends StorageBackend {
 
  /// Not part of public API
  @visibleForTesting
- dynamic decodeValue(dynamic value) {
+ Future<dynamic> decodeValue(dynamic value) async {
  if (value is ByteBuffer) {
  var bytes = Uint8List.view(value);
  if (_isEncoded(bytes)) {
@@ -131,9 +131,9 @@ class StorageBackendJs extends StorageBackend {
  if (hasProperty(store, 'getAll') && !cursor) {
  var completer = Completer<Iterable<dynamic>>();
  var request = store.getAll(null);
- request.onSuccess.listen((_) {
- var values = (request.result as List).map(decodeValue);
- completer.complete(values);
+ request.onSuccess.listen((_) async {
+ var futures = (request.result as List).map(decodeValue);
+ completer.complete(await Future.wait(futures));
  });
  request.onError.listen((_) {
  completer.completeError(request.error!);
@@ -178,7 +178,7 @@ class StorageBackendJs extends StorageBackend {
  if (frame.deleted) {
  await store.delete(frame.key);
  } else {
- await store.put(encodeValue(frame), frame.key);
+ await store.put(await encodeValue(frame), frame.key);
  }
  }
  }

hive/lib/src/backend/storage_backend_memory.dart

@@ -27,8 +27,8 @@ class StorageBackendMemory extends StorageBackend {
 
  @override
  Future<void> initialize(
- TypeRegistry registry, Keystore? keystore, bool lazy) {
- var recoveryOffset = _frameHelper.framesFromBytes(
+ TypeRegistry registry, Keystore? keystore, bool lazy) async {
+ var recoveryOffset = await _frameHelper.framesFromBytes(
  _bytes!, // Initialized at constructor and nulled after initialization
  keystore,
  registry,

hive/lib/src/backend/vm/storage_backend_vm.dart

@@ -101,14 +101,14 @@ class StorageBackendVm extends StorageBackend {
  }
 
  @override
- Future<dynamic> readValue(Frame frame) {
- return _sync.syncRead(() async {
+ Future<dynamic> readValue(Frame frame) async {
+ return await _sync.syncRead(() async {
  await readRaf.setPosition(frame.offset);
 
  var bytes = await readRaf.read(frame.length!);
 
  var reader = BinaryReaderImpl(bytes, registry);
- var readFrame = reader.readFrame(cipher: _cipher, lazy: false);
+ var readFrame = await reader.readFrame(cipher: _cipher, lazy: false);
 
  if (readFrame == null) {
  throw HiveError(
@@ -120,34 +120,37 @@ class StorageBackendVm extends StorageBackend {
  }
 
  @override
- Future<void> writeFrames(List<Frame> frames) {
- return _sync.syncWrite(() async {
- var writer = BinaryWriterImpl(registry);
+ Future<void> writeFrames(List<Frame> frames) async {
+ var writer = BinaryWriterImpl(registry);
 
- for (var frame in frames) {
- frame.length = writer.writeFrame(frame, cipher: _cipher);
- }
+ for (var frame in frames) {
+ frame.length = await writer.writeFrame(frame, cipher: _cipher);
+ }
+ await _sync.syncWrite(() async {
+ final bytes = writer.toBytes();
 
+ final cachedOffset = writeOffset;
  try {
- await writeRaf.writeFrom(writer.toBytes());
+ /// TODO(TheOneWithTheBraid): implement real transactions with cache
+ await writeRaf.writeFrom(bytes);
  } catch (e) {
- await writeRaf.setPosition(writeOffset);
+ await writeRaf.setPosition(cachedOffset);
  rethrow;
  }
-
- for (var frame in frames) {
- frame.offset = writeOffset;
- writeOffset += frame.length!;
- }
  });
+
+ for (var frame in frames) {
+ frame.offset = writeOffset;
+ writeOffset += frame.length!;
+ }
  }
 
  @override
- Future<void> compact(Iterable<Frame> frames) {
+ Future<void> compact(Iterable<Frame> frames) async {
  if (_compactionScheduled) return Future.value();
  _compactionScheduled = true;
 
- return _sync.syncReadWrite(() async {
+ await _sync.syncReadWrite(() async {
  await readRaf.setPosition(0);
  var reader = BufferedFileReader(readRaf);
 
@@ -216,7 +219,7 @@ class StorageBackendVm extends StorageBackend {
  }
 
  @override
- Future<void> close() {
+ Future<void> close() async {
  return _sync.syncReadWrite(_closeInternal);
  }
 
@@ -229,8 +232,8 @@ class StorageBackendVm extends StorageBackend {
  }
 
  @override
- Future<void> flush() {
- return _sync.syncWrite(() async {
+ Future<void> flush() async {
+ await _sync.syncWrite(() async {
  await writeRaf.flush();
  });
  }

hive/lib/src/binary/binary_reader_impl.dart

@@ -241,8 +241,8 @@ class BinaryReaderImpl extends BinaryReader {
  }
 
  /// Not part of public API
- Frame? readFrame(
- {HiveCipher? cipher, bool lazy = false, int frameOffset = 0}) {
+ Future<Frame?> readFrame(
+ {HiveCipher? cipher, bool lazy = false, int frameOffset = 0}) async {
  // frame length is stored on 4 bytes
  if (availableBytes < 4) return null;
 
@@ -275,7 +275,7 @@ class BinaryReaderImpl extends BinaryReader {
  } else if (cipher == null) {
  frame = Frame(key, read());
  } else {
- frame = Frame(key, readEncrypted(cipher));
+ frame = Frame(key, await readEncrypted(cipher));
  }
 
  frame
@@ -332,10 +332,10 @@ class BinaryReaderImpl extends BinaryReader {
  /// Not part of public API
  @pragma('vm:prefer-inline')
  @pragma('dart2js:tryInline')
- dynamic readEncrypted(HiveCipher cipher) {
+ Future<dynamic> readEncrypted(HiveCipher cipher) async {
  var inpLength = availableBytes;
  var out = Uint8List(inpLength);
- var outLength = cipher.decrypt(_buffer, _offset, inpLength, out, 0);
+ var outLength = await cipher.decrypt(_buffer, _offset, inpLength, out, 0);
  _offset += inpLength;
 
  var valueReader = BinaryReaderImpl(out, _typeRegistry, outLength);

hive/lib/src/binary/binary_writer_impl.dart

@@ -1,3 +1,4 @@
+import 'dart:async';
 import 'dart:convert';
 import 'dart:typed_data';
 
@@ -268,7 +269,7 @@ class BinaryWriterImpl extends BinaryWriter {
  }
 
  /// Not part of public API
- int writeFrame(Frame frame, {HiveCipher? cipher}) {
+ Future<int> writeFrame(Frame frame, {HiveCipher? cipher}) async {
  ArgumentError.checkNotNull(frame);
 
  var startOffset = _offset;
@@ -281,7 +282,7 @@ class BinaryWriterImpl extends BinaryWriter {
  if (cipher == null) {
  write(frame.value);
  } else {
- writeEncrypted(frame.value, cipher);
+ await writeEncrypted(frame.value, cipher);
  }
  }
 
@@ -394,16 +395,16 @@ class BinaryWriterImpl extends BinaryWriter {
  /// Not part of public API
  @pragma('vm:prefer-inline')
  @pragma('dart2js:tryInline')
- void writeEncrypted(dynamic value, HiveCipher cipher,
- {bool writeTypeId = true}) {
+ FutureOr<void> writeEncrypted(dynamic value, HiveCipher cipher,
+ {bool writeTypeId = true}) async {
  var valueWriter = BinaryWriterImpl(_typeRegistry)
  ..write(value, writeTypeId: writeTypeId);
  var inp = valueWriter._buffer;
  var inpLength = valueWriter._offset;
 
  _reserveBytes(cipher.maxEncryptedSize(inp));
 
- var len = cipher.encrypt(inp, 0, inpLength, _buffer, _offset);
+ var len = await cipher.encrypt(inp, 0, inpLength, _buffer, _offset);
 
  _offset += len;
  }

hive/lib/src/binary/frame_helper.dart

@@ -7,18 +7,18 @@ import 'package:hive/src/box/keystore.dart';
 /// Not part of public API
 class FrameHelper {
  /// Not part of public API
- int framesFromBytes(
+ Future<int> framesFromBytes(
  Uint8List bytes,
  Keystore? keystore,
  TypeRegistry registry,
  HiveCipher? cipher,
- ) {
+ ) async {
  var reader = BinaryReaderImpl(bytes, registry);
 
  while (reader.availableBytes != 0) {
  var frameOffset = reader.usedBytes;
 
- var frame = reader.readFrame(
+ var frame = await reader.readFrame(
  cipher: cipher,
  lazy: false,
  frameOffset: frameOffset,

hive/lib/src/box/box_base.dart

@@ -112,6 +112,12 @@ abstract class BoxBase<E> {
  bool notify = true,
  });
 
+ /// Beta: Performs the following write-only operations in memory and later
+ /// flushes
+ ///
+ /// This can be used to improve speed on many small write operations
+ // Future<void> transaction(Future<void> Function() operation);
+
  /// Deletes the given [key] from the box.
  ///
  /// If it does not exist, nothing happens.

hive/lib/src/box/box_base_impl.dart

@@ -1,3 +1,5 @@
+import 'dart:async';
+
 import 'package:hive/hive.dart';
 import 'package:hive/src/backend/storage_backend.dart';
 import 'package:hive/src/box/change_notifier.dart';
@@ -176,25 +178,30 @@ abstract class BoxBaseImpl<E> implements BoxBase<E> {
  if (!backend.supportsCompaction) return;
  if (keystore.deletedEntries == 0) return;
 
+ await flush();
+
  await backend.compact(keystore.frames);
  keystore.resetDeletedEntries();
  }
 
  /// Not part of public API
  @protected
- Future<void> performCompactionIfNeeded() {
+ FutureOr<void> performCompactionIfNeeded() {
  if (_compactionStrategy(keystore.length, keystore.deletedEntries)) {
  return compact();
  }
-
- return Future.value();
  }
 
  @override
  Future<void> close() async {
  if (!_open) return;
 
- _open = false;
+ try {
+ await flush();
+ } finally {
+ _open = false;
+ }
+
  await keystore.close();
  hive.unregisterBox(name);
 

hive/lib/src/box_collection/box_collection.dart

@@ -240,12 +240,11 @@ class CollectionBox<V> implements implementation.CollectionBox<V> {
 
  @override
  Future<void> flush() async {
- final box = await _getBox();
  // we do *not* await the flushing here. That makes it so that we can execute
  // other stuff while the flushing is still in progress. Fortunately, hive
  // has a proper read / write queue, meaning that if we do actually want to
  // write something again, it'll wait until the flush is completed.
- box.flush();
+ _getBox().then((box) => box.flush());
  }
 
  Future<void> _flushOrMark() async {

hive/lib/src/crypto/hive_aes_cipher.dart

@@ -24,7 +24,7 @@ class HiveAesCipher implements HiveCipher {
  int calculateKeyCrc() => _keyCrc;
 
  @override
- int decrypt(
+ FutureOr<int> decrypt(
  Uint8List inp, int inpOff, int inpLength, Uint8List out, int outOff) {
  var iv = inp.view(inpOff, 16);
 
@@ -36,7 +36,7 @@ class HiveAesCipher implements HiveCipher {
  Uint8List generateIv() => _ivRandom.nextBytes(16);
 
  @override
- int encrypt(
+ FutureOr<int> encrypt(
  Uint8List inp, int inpOff, int inpLength, Uint8List out, int outOff) {
  var iv = generateIv();
  out.setAll(outOff, iv);