Skip to content

Commit

Permalink
Application: Fix leaky Oodle abstraction; make tests independent from it
Browse files Browse the repository at this point in the history
  • Loading branch information
ShadelessFox committed May 18, 2024
1 parent 84b1c8f commit 6bf31e6
Show file tree
Hide file tree
Showing 12 changed files with 217 additions and 244 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import com.shade.decima.model.app.impl.DSPackfileProvider;
import com.shade.decima.model.app.impl.HZDPackfileProvider;
import com.shade.decima.model.base.CoreBinary;
import com.shade.decima.model.packfile.Oodle;
import com.shade.decima.model.packfile.Packfile;
import com.shade.decima.model.packfile.PackfileManager;
import com.shade.decima.model.packfile.PackfileProvider;
Expand All @@ -12,7 +13,7 @@
import com.shade.decima.model.rtti.RTTICoreFileReader.ThrowingErrorHandlingStrategy;
import com.shade.decima.model.rtti.objects.RTTIObject;
import com.shade.decima.model.rtti.registry.RTTITypeRegistry;
import com.shade.decima.model.util.Oodle;
import com.shade.decima.model.util.Compressor;
import com.shade.platform.model.util.IOUtils;
import com.shade.util.NotNull;
import com.shade.util.Nullable;
Expand All @@ -36,14 +37,14 @@ public class Project implements Closeable {
private final RTTITypeRegistry typeRegistry;
private final RTTICoreFileReader coreFileReader;
private final PackfileManager packfileManager;
private final Oodle oodle;
private final Oodle compressor;

Project(@NotNull ProjectContainer container) throws IOException {
this.container = container;
this.typeRegistry = new RTTITypeRegistry(container);
this.coreFileReader = new CoreBinary.Reader(typeRegistry);
this.oodle = Oodle.acquire(container.getCompressorPath());
this.packfileManager = new PackfileManager(oodle);
this.compressor = Oodle.acquire(container.getCompressorPath());
this.packfileManager = new PackfileManager(compressor);

mountDefaults();
}
Expand Down Expand Up @@ -89,8 +90,8 @@ public PackfileManager getPackfileManager() {
}

@NotNull
public Oodle getCompressor() {
return oodle;
public Compressor getCompressor() {
return compressor;
}

@NotNull
Expand Down Expand Up @@ -145,7 +146,7 @@ public Map<Long, long[]> listFileLinks() throws IOException {
@Override
public void close() throws IOException {
packfileManager.close();
oodle.close();
compressor.close();
}

@NotNull
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
package com.shade.decima.model.packfile;

import com.shade.decima.model.util.CloseableLibrary;
import com.shade.decima.model.util.Compressor;
import com.shade.platform.model.util.MathUtils;
import com.shade.util.NotNull;

import java.io.Closeable;
import java.io.IOException;
import java.lang.ref.Reference;
import java.lang.ref.WeakReference;
import java.nio.ByteBuffer;
import java.nio.file.Path;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;

public class Oodle implements Compressor, Closeable {
private static final Map<Path, Reference<Oodle>> compressors = new ConcurrentHashMap<>();

private final OodleLibrary library;
private final Path path;
private volatile int useCount;

private Oodle(@NotNull Path path) {
this.library = CloseableLibrary.load(path.toString(), OodleLibrary.class);
this.path = path;
this.useCount = 1;
}

@NotNull
public static Oodle acquire(@NotNull Path path) {
final Reference<Oodle> ref = compressors.get(path);
Oodle oodle = ref != null ? ref.get() : null;

if (oodle == null) {
oodle = new Oodle(path);
compressors.put(path, new WeakReference<>(oodle));
} else {
synchronized (oodle.library) {
oodle.useCount += 1;
}
}

return oodle;
}

@NotNull
@Override
public ByteBuffer compress(@NotNull ByteBuffer src, @NotNull Level level) throws IOException {
final var dst = ByteBuffer.allocate(getCompressedSize(src.remaining()));
final var len = library.OodleLZ_Compress(8, src, src.remaining(), dst, getCompressionLevel(level), 0, 0, 0, 0, 0);
if (len == 0) {
throw new IOException("Error compressing data");
}
return dst.slice(0, len);
}

@Override
public void decompress(@NotNull ByteBuffer src, @NotNull ByteBuffer dst) throws IOException {
final int len = library.OodleLZ_Decompress(src, src.remaining(), dst, dst.remaining(), 1, 0, 0, 0, 0, 0, 0, 0, 0, 3);
if (len != dst.remaining()) {
throw new IOException("Error decompressing data");
}
}

public int getVersion() {
final int[] buffer = new int[7];
library.Oodle_GetConfigValues(buffer);
return buffer[6];
}

@NotNull
public String getVersionString() {
final int version = getVersion();
// The packed data is:
// (46 << 24) | (OODLE2_VERSION_MAJOR << 16) | (OODLE2_VERSION_MINOR << 8) | sizeof(OodleLZ_SeekTable)
// The version string is:
// 2 . OODLE2_VERSION_MAJOR . OODLE2_VERSION_MINOR
return String.format("2.%d.%d", version >>> 16 & 0xff, version >>> 8 & 0xff);
}

private static int getCompressedSize(int size) {
return size + 274 * MathUtils.ceilDiv(size, 0x40000);
}

private static int getCompressionLevel(@NotNull Level level) {
return switch (level) {
case NONE -> /* NONE */ 0;
case FAST -> /* SUPER_FAST */ 1;
case NORMAL -> /* NORMAL */ 4;
case BEST -> /* OPTIMAL_5 */ 9;
};
}

@Override
public void close() {
synchronized (library) {
if (useCount <= 0) {
throw new IllegalStateException("Compressor is disposed");
}

useCount -= 1;

if (useCount == 0) {
library.close();
compressors.remove(path);
}
}
}

@Override
public String toString() {
return "Compressor{path=" + path + ", version=" + getVersionString() + '}';
}

private interface OodleLibrary extends CloseableLibrary {
int OodleLZ_Compress(int compressor, ByteBuffer rawBuf, long rawLen, ByteBuffer compBuf, int level, long pOptions, long dictionaryBase, long lrm, long scratchMem, long scratchSize);

int OodleLZ_Decompress(ByteBuffer compBuf, long compBufSize, ByteBuffer rawBuf, long rawLen, int fuzzSafe, int checkCRC, int verbosity, long decBufBase, long decBufSize, long fpCallback, long callbackUserData, long decoderMemory, long decoderMemorySize, int threadPhase);

void Oodle_GetConfigValues(int[] buffer);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@
import com.shade.decima.model.archive.ArchiveFile;
import com.shade.decima.model.packfile.edit.Change;
import com.shade.decima.model.packfile.resource.Resource;
import com.shade.decima.model.util.Compressor;
import com.shade.decima.model.util.FilePath;
import com.shade.decima.model.util.Oodle;
import com.shade.decima.model.util.hash.MurmurHash3;
import com.shade.platform.model.messages.MessageBus;
import com.shade.platform.model.messages.Topic;
Expand Down Expand Up @@ -34,9 +34,10 @@ public class Packfile implements Archive, Comparable<Packfile> {
public static final long[] DATA_KEY = {0x7E159D956C084A37L, 0x18AA7D3F3D5AF7E8L};
public static final int MAGIC_PLAIN = 0x20304050;
public static final int MAGIC_ENCRYPTED = 0x21304050;
public static final int MAXIMUM_BLOCK_SIZE = 0x40000;

private final PackfileManager manager;
private final Oodle oodle;
private final Compressor compressor;
private final PackfileInfo info;
private final Map<FilePath, Change> changes = new HashMap<>();

Expand All @@ -47,9 +48,9 @@ public class Packfile implements Archive, Comparable<Packfile> {
private final NavigableMap<Long, ChunkEntry> chunks = new TreeMap<>(Long::compareUnsigned);
private SeekableByteChannel channel;

Packfile(@NotNull PackfileManager manager, @NotNull Oodle oodle, @NotNull PackfileInfo info) throws IOException {
Packfile(@NotNull PackfileManager manager, @NotNull Compressor compressor, @NotNull PackfileInfo info) throws IOException {
this.manager = manager;
this.oodle = oodle;
this.compressor = compressor;
this.info = info;

read();
Expand Down Expand Up @@ -372,8 +373,8 @@ protected void validate() throws IOException {
throw new IOException("Data size does not match the actual size (expected: " + actualDataSize + ", actual: " + header.dataSize() + ")");
}

if (Oodle.BLOCK_SIZE_BYTES != header.chunkEntrySize()) {
throw new IOException("Unexpected maximum chunk size (expected: " + Oodle.BLOCK_SIZE_BYTES + ", actual: " + header.chunkEntrySize() + ")");
if (MAXIMUM_BLOCK_SIZE != header.chunkEntrySize()) {
throw new IOException("Unexpected maximum chunk size (expected: " + MAXIMUM_BLOCK_SIZE + ", actual: " + header.chunkEntrySize() + ")");
}

Span lastCompressedSpan = null;
Expand Down Expand Up @@ -667,7 +668,6 @@ private class PackfileInputStream extends InputStream {
private final FileEntry file;
private final ChunkEntry[] chunks;

private final byte[] compressed = new byte[Oodle.getCompressedSize(header.chunkEntrySize())];
private final byte[] decompressed = new byte[header.chunkEntrySize()];

private int index; // index of the current chunk
Expand Down Expand Up @@ -735,7 +735,7 @@ private void fill() throws IOException {
}

final ChunkEntry chunk = chunks[index];
final ByteBuffer buffer = ByteBuffer.wrap(compressed, 0, chunk.compressed().size());
final ByteBuffer buffer = ByteBuffer.allocate(chunk.compressed().size());

synchronized (Packfile.this) {
channel.position(chunk.compressed().offset());
Expand All @@ -746,7 +746,7 @@ private void fill() throws IOException {
chunk.swizzle(buffer.slice());
}

oodle.decompress(compressed, chunk.compressed().size(), decompressed, chunk.decompressed().size());
compressor.decompress(buffer, ByteBuffer.wrap(decompressed));

if (index == 0) {
pos = (int) (file.span().offset() - chunk.decompressed().offset());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
import com.shade.decima.model.archive.ArchiveFile;
import com.shade.decima.model.archive.ArchiveManager;
import com.shade.decima.model.packfile.edit.Change;
import com.shade.decima.model.util.Compressor;
import com.shade.decima.model.util.FilePath;
import com.shade.decima.model.util.Oodle;
import com.shade.platform.model.util.IOUtils;
import com.shade.util.NotNull;
import com.shade.util.Nullable;
Expand All @@ -22,18 +22,18 @@ public class PackfileManager implements ArchiveManager {
private static final Logger log = LoggerFactory.getLogger(PackfileManager.class);

private final NavigableSet<Packfile> packfiles = new TreeSet<>();
private final Oodle oodle;
private final Compressor compressor;

public PackfileManager(@NotNull Oodle oodle) {
this.oodle = oodle;
public PackfileManager(@NotNull Compressor compressor) {
this.compressor = compressor;
}

public void mountPackfile(@NotNull PackfileInfo info) throws IOException {
if (Files.notExists(info.path())) {
return;
}

final Packfile packfile = new Packfile(this, oodle, info);
final Packfile packfile = new Packfile(this, compressor, info);

synchronized (this) {
if (!packfiles.add(packfile)) {
Expand All @@ -47,7 +47,7 @@ public void mountPackfile(@NotNull PackfileInfo info) throws IOException {

@NotNull
public Packfile openPackfile(@NotNull Path path) throws IOException {
return new Packfile(this, oodle, new PackfileInfo(path, IOUtils.getBasename(path), null));
return new Packfile(this, compressor, new PackfileInfo(path, IOUtils.getBasename(path), null));
}

@Nullable
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
package com.shade.decima.model.packfile;

import com.shade.decima.model.packfile.resource.Resource;
import com.shade.decima.model.util.Oodle;
import com.shade.decima.model.util.Compressor;
import com.shade.platform.model.runtime.ProgressMonitor;
import com.shade.platform.model.util.MathUtils;
import com.shade.util.NotNull;

import java.io.Closeable;
Expand All @@ -28,7 +29,7 @@ public boolean add(@NotNull Resource resource) {
public long write(
@NotNull ProgressMonitor monitor,
@NotNull SeekableByteChannel channel,
@NotNull Oodle oodle,
@NotNull Compressor compressor,
@NotNull Options options
) throws IOException {
final RandomGenerator random = new SecureRandom();
Expand All @@ -37,7 +38,7 @@ public long write(

try (ProgressMonitor.Task task = monitor.begin("Write packfile", 2)) {
channel.position(computeHeaderSize());
writeData(task.split(1), channel, oodle, random, options, files, chunks);
writeData(task.split(1), channel, compressor, random, options, files, chunks);

channel.position(0);
return writeHeader(task.split(1), channel, random, options, files, chunks).fileSize();
Expand Down Expand Up @@ -74,7 +75,7 @@ private Packfile.Header writeHeader(
decompressedSize,
files.size(),
chunks.size(),
Oodle.BLOCK_SIZE_BYTES
Packfile.MAXIMUM_BLOCK_SIZE
);

try (ProgressMonitor.Task task = monitor.begin("Write header", 1)) {
Expand All @@ -98,14 +99,14 @@ private Packfile.Header writeHeader(
private void writeData(
@NotNull ProgressMonitor monitor,
@NotNull SeekableByteChannel channel,
@NotNull Oodle oodle,
@NotNull Compressor compressor,
@NotNull RandomGenerator random,
@NotNull Options options,
@NotNull Set<Packfile.FileEntry> files,
@NotNull Set<Packfile.ChunkEntry> chunks
) throws IOException {
final Queue<Resource> pending = new ArrayDeque<>(resources);
final ByteBuffer decompressed = ByteBuffer.allocate(Oodle.BLOCK_SIZE_BYTES);
final ByteBuffer decompressed = ByteBuffer.allocate(Packfile.MAXIMUM_BLOCK_SIZE);

long fileDataOffset = 0;
long chunkDataDecompressedOffset = 0;
Expand Down Expand Up @@ -151,7 +152,7 @@ private void writeData(
decompressed.limit(decompressed.position());
decompressed.position(0);

final ByteBuffer compressed = oodle.compress(decompressed.slice(), options.compression());
final ByteBuffer compressed = compressor.compress(decompressed.slice(), options.compression());

final Packfile.Span decompressedSpan = new Packfile.Span(
chunkDataDecompressedOffset,
Expand Down Expand Up @@ -203,8 +204,8 @@ private int computeChunksCount() {
.mapToLong(Resource::size)
.sum();

return Math.max(1, Oodle.getBlocksCount(size));
return Math.max(1, Math.toIntExact(MathUtils.ceilDiv(size, Packfile.MAXIMUM_BLOCK_SIZE)));
}

public record Options(@NotNull Oodle.CompressionLevel compression, boolean encrypt) {}
public record Options(@NotNull Compressor.Level compression, boolean encrypt) {}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
package com.shade.decima.model.util;

import com.shade.util.NotNull;

import java.io.IOException;
import java.nio.ByteBuffer;

public interface Compressor {
enum Level {
NONE,
FAST,
NORMAL,
BEST
}

@NotNull
ByteBuffer compress(@NotNull ByteBuffer src, @NotNull Level level) throws IOException;

void decompress(@NotNull ByteBuffer src, @NotNull ByteBuffer dst) throws IOException;
}
Loading

0 comments on commit 6bf31e6

Please sign in to comment.