Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adding support for named graphs #196

Merged
merged 32 commits into from
Sep 14, 2023
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
f69df74
Implemented quad dictionaries for .nq files
QuentinJanuel May 31, 2023
728446d
temp triples can handle quads
QuentinJanuel Jun 1, 2023
3d9e29a
making sure graphs don't affect normal hdt
QuentinJanuel Jun 5, 2023
9a7d09d
FourQuadSectionDictionary needs to implement FourSectionDictionary, n…
QuentinJanuel Jun 15, 2023
7a024f5
quad patterns support (not implemented yet)
QuentinJanuel Jun 15, 2023
82cb465
fixed quadstring
QuentinJanuel Jun 22, 2023
3439c49
save triples (could be optimized)
QuentinJanuel Jun 22, 2023
a4f5bfb
search wip
QuentinJanuel Jun 26, 2023
d672ee9
Fixed search for triples (non quad)
QuentinJanuel Jun 29, 2023
c26b85e
number of graphs is now dynamic
QuentinJanuel Jun 29, 2023
f58cfa6
Adapt code for large files (RoaringBitmap)
QuentinJanuel Jul 11, 2023
bcc095a
roaring bitmap 64
QuentinJanuel Jul 13, 2023
f45facb
accelerated wildcard pattern a bit
QuentinJanuel Jul 13, 2023
1e612c3
BitmapQuadsIterator
QuentinJanuel Jul 17, 2023
8d31b3d
fixed roaring bitmap
QuentinJanuel Jul 20, 2023
c8b0c1e
let quads iterator variables be longs
QuentinJanuel Jul 20, 2023
b1ba3a6
support for ???G, S??G, SP?G & SPOG
QuentinJanuel Jul 20, 2023
0f1685b
?POG & ??OG (ZGFOQ)
QuentinJanuel Jul 28, 2023
5ccc86b
removed forgotten log
QuentinJanuel Jul 28, 2023
257aa30
?PO? & ??O? (ZFOQ)
QuentinJanuel Jul 28, 2023
76ee129
?P?G (YGFOQ)
QuentinJanuel Jul 31, 2023
41abd02
?P?? (YFOQ)
QuentinJanuel Aug 4, 2023
cbfeedc
address https://github.com/rdfhdt/hdt-java/pull/196#discussion_r12416…
QuentinJanuel Aug 28, 2023
b9002a6
address https://github.com/rdfhdt/hdt-java/pull/196#discussion_r12955…
QuentinJanuel Aug 28, 2023
fef55fb
address https://github.com/rdfhdt/hdt-java/pull/196#discussion_r12955…
QuentinJanuel Aug 28, 2023
5233663
address https://github.com/rdfhdt/hdt-java/pull/196#discussion_r12955…
QuentinJanuel Aug 28, 2023
59bba7c
address https://github.com/rdfhdt/hdt-java/pull/196#discussion_r12955…
QuentinJanuel Aug 28, 2023
7ae491c
address https://github.com/rdfhdt/hdt-java/pull/196#discussion_r12958…
QuentinJanuel Aug 28, 2023
eacd5bb
address https://github.com/rdfhdt/hdt-java/pull/196#discussion_r12958…
QuentinJanuel Aug 28, 2023
f61bc58
address https://github.com/rdfhdt/hdt-java/pull/196#discussion_r12959…
QuentinJanuel Aug 31, 2023
c675a01
cat tests for quads
QuentinJanuel Sep 4, 2023
3f4ba41
test quads iterators
QuentinJanuel Sep 5, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
?POG & ??OG (ZGFOQ)
  • Loading branch information
QuentinJanuel committed Jul 28, 2023
commit 0f1685b3ddf1cfb0898a62e526257977a77aa343
Original file line number Diff line number Diff line change
@@ -0,0 +1,179 @@
package org.rdfhdt.hdt.quads.impl;

import org.rdfhdt.hdt.compact.bitmap.Bitmap;
import org.rdfhdt.hdt.enums.ResultEstimationType;
import org.rdfhdt.hdt.enums.TripleComponentOrder;
import org.rdfhdt.hdt.exceptions.NotImplementedException;
import org.rdfhdt.hdt.triples.TripleID;
import org.rdfhdt.hdt.triples.impl.BitmapTriples;
import org.rdfhdt.hdt.triples.impl.BitmapTriplesIteratorZFOQ;
import org.rdfhdt.hdt.triples.impl.TripleOrderConvert;

public class BitmapQuadsIteratorZGFOQ extends BitmapTriplesIteratorZFOQ{
// resolves ?POG, ??OG queries

private Bitmap bitmapGraph; // the bitmap of the requested graph

public BitmapQuadsIteratorZGFOQ(BitmapTriples triples, TripleID pattern) {
super(triples, pattern);

System.out.println("ZGFOQ");

this.triples = triples;
this.pattern = new TripleID();
this.returnTriple = new TripleID();
this.bitmapGraph = triples
.getQuadInfoAG()
.get((int) pattern.getGraph() - 1);
newSearch(pattern);
}

protected void findRange() {
findRange2();
while(maxIndex >= minIndex && !bitmapGraph.access(getTriplePosition(maxIndex))) {
maxIndex--;
}

while(maxIndex >= minIndex && !bitmapGraph.access(getTriplePosition(minIndex))) {
minIndex++;
}
}

/*
* Check if there are more solution
*/
@Override
public boolean hasNext() {
return posIndex<=maxIndex && maxIndex >= minIndex;
}

/*
* Get the next solution
*/
@Override
public TripleID next() {
long posY = adjIndex.get(posIndex); // get the position of the next occurrence of the predicate in AdjY

z = patZ!=0 ? patZ : (int)adjIndex.findListIndex(posIndex)+1; //get the next object (z) as the number of list in adIndex corresponding to posIndex
y = patY!=0 ? patY : (int) adjY.get(posY); // get the next predicate (y) as the element in adjY stores in position posY
x = (int) adjY.findListIndex(posY)+1; //get the next subject (X) as the number of list in adjY corresponding to posY

do {
posIndex++; // increase the position of the next occurrence of the predicate
} while(posIndex < maxIndex && !bitmapGraph.access(getNextTriplePosition()));

updateOutput(); // set the components (subject,predicate,object) of the returned triple
return returnTriple; // return the triple as solution
}

public long getTriplePosition(long index) {
long ret =0;
try {
ret = triples.adjZ.find(adjIndex.get(index),patZ);
} catch (Exception e) {}
return ret;
}

protected void newSearch(TripleID pattern) {
this.pattern.assign(pattern);

TripleOrderConvert.swapComponentOrder(this.pattern, TripleComponentOrder.SPO, triples.order);
patZ = this.pattern.getObject();
if(patZ==0 && (patY!=0 || this.pattern.getSubject()!=0)) {
throw new IllegalArgumentException("This structure is not meant to process this pattern");
}

patY = this.pattern.getPredicate();

adjY = triples.adjY;
adjIndex = triples.adjIndex; // adjIndex has the list of positions in adY

findRange(); // get the boundaries where the solution for the given object can be found
goToStart(); // load the first solution and position the next pointers
}

protected void findRange2() {
if(patZ==0) { //if the object is not provided (usually it is in this iterator)
minIndex = 0;
maxIndex = adjIndex.getNumberOfElements();
return;
}
minIndex = adjIndex.find(patZ-1); //find the position of the first occurrence of the object
maxIndex = adjIndex.last(patZ-1); //find the position of the last ocurrence of the object

if(patY!=0) { // if the predicate is provided then we do a binary search to search for such predicate
while (minIndex <= maxIndex) {
long mid = (minIndex + maxIndex) / 2;
long predicate = getY(mid); //get predicate at mid position in the object index
if (patY > predicate) {
minIndex = mid + 1;
} else if (patY < predicate) {
maxIndex = mid - 1;
} else { // the predicate has been found, now we have to find the min and max limits (the predicate P is repeated for each PO occurrence in the triples)
// Binary Search to find left boundary
long left=minIndex;
long right=mid;
long pos=0;

while(left<=right) {
pos = (left+right)/2;

predicate = getY(pos);

if(predicate!=patY) {
left = pos+1;
} else {
right = pos-1;
}
}
minIndex = predicate==patY ? pos : pos+1;
// Binary Search to find right boundary
left = mid;
right= maxIndex;

while(left<=right) {
pos = (left+right)/2;
predicate = getY(pos);

if(predicate!=patY) {
right = pos-1;
} else {
left = pos+1;
}
}
maxIndex = predicate==patY ? pos : pos-1;
break;
}
}
}
}

public long getNextTriplePosition() {
long ret =0;
try {
ret = triples.adjZ.find(adjIndex.get(posIndex),patZ);
} catch (Exception e) { }
return ret;
}


@Override
public ResultEstimationType numResultEstimation() {
return ResultEstimationType.UP_TO;
}

@Override
public boolean hasPrevious() {
throw new NotImplementedException();
}

@Override
public TripleID previous() {
throw new NotImplementedException();
}

@Override
public void goTo(long pos) {
throw new NotImplementedException();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@
import org.rdfhdt.hdt.options.HDTOptions;
import org.rdfhdt.hdt.quads.impl.BitmapQuadsIterator;
import org.rdfhdt.hdt.quads.impl.BitmapQuadsIteratorG;
import org.rdfhdt.hdt.quads.impl.BitmapQuadsIteratorZGFOQ;
import org.rdfhdt.hdt.triples.IteratorTripleID;
import org.rdfhdt.hdt.triples.TempTriples;
import org.rdfhdt.hdt.triples.TripleID;
Expand Down Expand Up @@ -296,6 +297,9 @@ public SuppliableIteratorTripleID search(TripleID pattern) {
}
}

if(patternString.equals("?POG") || patternString.equals("??OG"))
return new BitmapQuadsIteratorZGFOQ(this, pattern);

SuppliableIteratorTripleID bitIt;
if (patternString.endsWith("G"))
bitIt = new BitmapQuadsIteratorG(this, pattern);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ public class BitmapTriples implements TriplesPrivate {
protected Sequence seqY, seqZ, indexZ, predicateCount;
public Bitmap bitmapY, bitmapZ, bitmapIndexZ;

protected AdjacencyList adjY, adjZ, adjIndex;
public AdjacencyList adjY, adjZ, adjIndex;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there a reason for this change?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, the class BitmapTriples is used by, for instance, BitmapQuadsIteratorZFOQ which needs to read these fields. This is done with a composition pattern, not inheritance, so protected won't work. The best I could to is creating getters (and setters maybe? didn't check if needed)


// Index for Y
public PredicateIndex predicateIndex;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,17 +39,17 @@
*
*/
public class BitmapTriplesIteratorZFOQ implements SuppliableIteratorTripleID {
private long lastPosIndex;
final BitmapTriples triples;
final TripleID pattern;
final TripleID returnTriple;
AdjacencyList adjY, adjIndex;
long posIndex, minIndex, maxIndex;
long x, y, z;
long patY;
final long patZ;
protected long lastPosIndex;
protected BitmapTriples triples;
protected TripleID pattern;
protected TripleID returnTriple;

protected AdjacencyList adjY, adjIndex;
protected long posIndex, minIndex, maxIndex;
protected long x, y, z;

protected long patY;
protected long patZ;

public BitmapTriplesIteratorZFOQ(BitmapTriples triples, TripleID pattern) {
this.triples = triples;
Expand All @@ -70,8 +70,8 @@ public BitmapTriplesIteratorZFOQ(BitmapTriples triples, TripleID pattern) {
calculateRange();
goToStart();
}
private long getY(long index) {

protected long getY(long index) {
return adjY.get(adjIndex.get(index));
}

Expand Down Expand Up @@ -134,7 +134,7 @@ private void calculateRange() {
}
}

private void updateOutput() {
protected void updateOutput() {
lastPosIndex = posIndex;
returnTriple.setAll(x, y, z);
TripleOrderConvert.swapComponentOrder(returnTriple, triples.order, TripleComponentOrder.SPO);
Expand Down