Skip to content

Commit

Permalink
Merge pull request #184 from ate47/dev_diffcat
Browse files Browse the repository at this point in the history
Add diffBit into k-HDTCat
  • Loading branch information
D063520 authored Dec 12, 2022
2 parents febc567 + 3b5908f commit ff0bcde
Show file tree
Hide file tree
Showing 29 changed files with 1,428 additions and 330 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,16 @@ public static Bitmap createBitmap(InputStream input) throws IOException {
return getInstance().doCreateBitmap(input);
}

/**
* @return an empty bitmap
*/
public static Bitmap empty() {
return getInstance().doEmpty();
}

// Abstract methods for the current implementation
protected abstract ModifiableBitmap doCreateModifiableBitmap(String type);
protected abstract ModifiableBitmap doCreateRWModifiableBitmap(long size);
protected abstract Bitmap doCreateBitmap(InputStream input) throws IOException;
protected abstract Bitmap doEmpty();
}
14 changes: 14 additions & 0 deletions hdt-api/src/main/java/org/rdfhdt/hdt/hdt/HDTManager.java
Original file line number Diff line number Diff line change
Expand Up @@ -501,6 +501,19 @@ public static HDT diffHDTBit(String location, String hdtFileName, Bitmap deleteB
return HDTManager.getInstance().doHDTDiffBit(location, hdtFileName, deleteBitmap, hdtFormat, listener);
}

/**
* Create an HDT file from HDT files by joining the triples and removing some triples with delete bitmaps
* @param hdtFileNames hdt file names
* @param deleteBitmaps the bitmaps for each HDT in hdtFileNames, should be the same size as hdtFileNames, see {@link org.rdfhdt.hdt.compact.bitmap.BitmapFactory#empty()}
* @param hdtFormat Parameters to tune the generated HDT.
* @param listener Listener to get notified of loading progress. Can be null if no notifications needed.
* @throws IOException when the file cannot be found
* @return HDT
*/
public static HDT diffBitCatHDT(List<String> hdtFileNames, List<? extends Bitmap> deleteBitmaps, HDTOptions hdtFormat, ProgressListener listener) throws IOException {
return HDTManager.getInstance().doHDTDiffBitCat(hdtFileNames, deleteBitmaps, hdtFormat, listener);
}


/**
* Create an HDT file from an RDF file in a tree, stop the chunk creation with the fluxStop
Expand Down Expand Up @@ -575,6 +588,7 @@ public static HDT catTree(RDFFluxStop fluxStop, HDTSupplier supplier, Iterator<T
protected abstract TripleWriter doGetHDTWriter(String outFile, String baseURI, HDTOptions hdtFormat) throws IOException;
protected abstract HDT doHDTCat(String location, String hdtFileName1, String hdtFileName2, HDTOptions hdtFormat, ProgressListener listener) throws IOException;
protected abstract HDT doHDTCat(List<String> hdtFileNames, HDTOptions hdtFormat, ProgressListener listener) throws IOException;
protected abstract HDT doHDTDiffBitCat(List<String> hdtFileNames, List<? extends Bitmap> deleteBitmaps, HDTOptions hdtFormat, ProgressListener listener) throws IOException;
protected abstract HDT doHDTDiff(String hdtFileName1, String hdtFileName2, HDTOptions hdtFormat, ProgressListener listener) throws IOException;
protected abstract HDT doHDTDiffBit(String location, String hdtFileName, Bitmap deleteBitmap, HDTOptions hdtFormat, ProgressListener listener) throws IOException;
protected abstract HDT doHDTCatTree(RDFFluxStop fluxStop, HDTSupplier supplier, String filename, String baseURI, RDFNotation rdfNotation, HDTOptions hdtFormat, ProgressListener listener) throws IOException, ParserException;
Expand Down
19 changes: 16 additions & 3 deletions hdt-api/src/main/java/org/rdfhdt/hdt/options/HDTOptions.java
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,9 @@ default RDFFluxStop getFluxStop(String key, RDFFluxStop defaultValue) {
* @param key key
* @return value or 0 if not defined
*/
long getInt(String key);
default long getInt(String key) {
return getInt(key, 0);
}

/**
* get a long
Expand Down Expand Up @@ -252,12 +254,23 @@ default void set(String key, Profiler profiler) {
* @param key key
* @param value value
*/
void setInt(String key, long value);
default void setInt(String key, long value) {
set(key, String.valueOf(value));
}

/**
* read an option config, format: (key=value)?(;key=value)*
*
* @param options options
*/
void setOptions(String options);
default void setOptions(String options) {
for (String item : options.split(";")) {
int pos = item.indexOf('=');
if (pos != -1) {
String property = item.substring(0, pos);
String value = item.substring(pos+1);
set(property, value);
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -199,13 +199,20 @@ public void set(long bitIndex, boolean value) {
throw new RuntimeException(e);
}

if(value) {
words.set(wordIndex, words.get(wordIndex) | (1L << bitIndex));
long wordText = words.get(wordIndex);
long wordReplaced;
if (value) {
wordReplaced = wordText | (1L << bitIndex);
} else {
words.set(wordIndex,words.get(wordIndex) & ~(1L << bitIndex));
wordReplaced = wordText & ~(1L << bitIndex);
}

this.numbits = Math.max(this.numbits, bitIndex+1);
if (wordText != wordReplaced) {
// we need to write something
words.set(wordIndex, wordReplaced);
}

this.numbits = Math.max(this.numbits, bitIndex + 1);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
*
*/
public class BitmapFactoryImpl extends BitmapFactory {
private final Bitmap EMPTY = new Bitmap64(0);

@Override
protected ModifiableBitmap doCreateModifiableBitmap(String type) {
Expand All @@ -63,4 +64,9 @@ protected Bitmap doCreateBitmap(InputStream input) throws IOException {
protected ModifiableBitmap doCreateRWModifiableBitmap(long size) {
return new Bitmap64(size);
}

@Override
protected Bitmap doEmpty() {
return EMPTY;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
package org.rdfhdt.hdt.compact.bitmap;

import org.rdfhdt.hdt.compact.integer.VByte;
import org.rdfhdt.hdt.exceptions.NotImplementedException;
import org.rdfhdt.hdt.hdt.HDTVocabulary;
import org.rdfhdt.hdt.listener.ProgressListener;
import org.rdfhdt.hdt.util.crc.CRC32;
import org.rdfhdt.hdt.util.crc.CRC8;
import org.rdfhdt.hdt.util.crc.CRCOutputStream;
import org.rdfhdt.hdt.util.io.IOUtil;

import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;

/**
* empty bitmap, act like a bitmap filled with 0, but isn't allocated on disk or in memory,
* will throw a {@link org.rdfhdt.hdt.exceptions.NotImplementedException} if we try to add
* a non 0 value
*/
public class EmptyBitmap implements ModifiableBitmap {
/**
* create empty bitmap simulating a bitmap of a particular size
*
* @param size the size
* @return bitmap
*/
public static ModifiableBitmap of(long size) {
return new EmptyBitmap(size);
}

private long size;

private EmptyBitmap(long size) {
this.size = size;
}

@Override
public void append(boolean value) {
set(size, value);
}

@Override
public void set(long pos, boolean value) {
if (value) {
throw new NotImplementedException("true value in EmptyBitmap");
}

size = Math.max(size, pos);
}

@Override
public boolean access(long pos) {
return false;
}

@Override
public long rank1(long pos) {
return 0;
}

@Override
public long rank0(long pos) {
return pos;
}

@Override
public long selectPrev1(long start) {
return -1;
}

@Override
public long selectNext1(long start) {
return -1;
}

@Override
public long select0(long n) {
return n;
}

@Override
public long select1(long n) {
return -1;
}

@Override
public long getNumBits() {
return size;
}

@Override
public long countOnes() {
return 0;
}

@Override
public long countZeros() {
return size;
}

@Override
public long getSizeBytes() {
return 0;
}

@Override
public void save(OutputStream output, ProgressListener listener) throws IOException {
CRCOutputStream out = new CRCOutputStream(output, new CRC8());

// Write Type and Numbits
out.write(BitmapFactory.TYPE_BITMAP_PLAIN);
VByte.encode(out, 8L);

// Write CRC
out.writeCRC();

// Setup new CRC
out.setCRC(new CRC32());
IOUtil.writeLong(out, 0);

out.writeCRC();
}

@Override
public void load(InputStream input, ProgressListener listener) {
throw new NotImplementedException();
}

@Override
public String getType() {
return HDTVocabulary.BITMAP_TYPE_PLAIN;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
package org.rdfhdt.hdt.compact.bitmap;

import org.rdfhdt.hdt.exceptions.NotImplementedException;
import org.rdfhdt.hdt.listener.ProgressListener;

import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;

/**
* create a negation bitmap of another one, the close method is called if the bitmap implements closeable
*
* access/set are negate when called
*
* @param <T> bitmap type
*/
public class NegBitmap<T extends Bitmap> implements Bitmap, Closeable {

/**
* create from a bitmap
*
* @param bitmap bitmap
* @return bitmap
*/
public static Bitmap of(Bitmap bitmap) {
return new NegBitmap<>(bitmap);
}

/**
* create from a modifiableBitmap
*
* @param modifiableBitmap modifiableBitmap
* @return modifiableBitmap
*/
public static ModifiableBitmap of(ModifiableBitmap modifiableBitmap) {
return new NegModifiableBitmap(modifiableBitmap);
}

private static class NegModifiableBitmap extends NegBitmap<ModifiableBitmap> implements ModifiableBitmap {

private NegModifiableBitmap(ModifiableBitmap handle) {
super(handle);
}

@Override
public void set(long position, boolean value) {
handle.set(position, !value);
}

@Override
public void append(boolean value) {
handle.append(!value);
}
}

protected final T handle;

private NegBitmap(T handle) {
this.handle = handle;
}

@Override
public boolean access(long position) {
return !handle.access(position);
}

@Override
public long rank1(long position) {
return handle.rank0(position);
}

@Override
public long rank0(long position) {
return handle.rank1(position);
}

@Override
public long selectPrev1(long start) {
throw new NotImplementedException();
}

@Override
public long selectNext1(long start) {
throw new NotImplementedException();
}

@Override
public long select0(long n) {
return handle.select1(n);
}

@Override
public long select1(long n) {
return handle.select0(n);
}

@Override
public long getNumBits() {
return handle.getNumBits();
}

@Override
public long countOnes() {
return handle.countZeros();
}

@Override
public long countZeros() {
return handle.countOnes();
}

@Override
public long getSizeBytes() {
return handle.getSizeBytes();
}

@Override
public void save(OutputStream output, ProgressListener listener) throws IOException {
throw new NotImplementedException();
}

@Override
public void load(InputStream input, ProgressListener listener) throws IOException {
throw new NotImplementedException();
}

@Override
public String getType() {
return handle.getType();
}

@Override
public void close() throws IOException {
if (handle instanceof Closeable) {
((Closeable) handle).close();
}
}
}
Loading

0 comments on commit ff0bcde

Please sign in to comment.