Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use logger instead of stdout/stderr outside of the the CLI #190

Merged
merged 1 commit into from
Feb 1, 2023
Merged

Use logger instead of stdout/stderr outside of the the CLI #190

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Use logger instead of stdout/stderr outside of the the CLI, fix log w… 
…ith Profiler #189
  • Loading branch information
@ate47
ate47 committed Feb 1, 2023
commit ca770d9ff73556a7f0a889b3892a898ef1892a66
2 changes: 1 addition & 1 deletion hdt-api/src/main/java/org/rdfhdt/hdt/util/Profiler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ public Profiler(String name, HDTOptions spec, boolean async) {
this.name = Objects.requireNonNull(name, "name can't be null!");
if (spec != null) {
String b = spec.get(async ? HDTOptionsKeys.PROFILER_ASYNC_KEY : HDTOptionsKeys.PROFILER_KEY);
disabled = b == null || b.length() == 0 || !(b.charAt(0) == '!' || "true".equalsIgnoreCase(b));
disabled = b == null || b.length() == 0 || !("true".equalsIgnoreCase(b));
String profilerOutputLocation = spec.get(async ? HDTOptionsKeys.PROFILER_ASYNC_OUTPUT_KEY : HDTOptionsKeys.PROFILER_OUTPUT_KEY);
if (profilerOutputLocation != null && !profilerOutputLocation.isEmpty()) {
outputPath = Path.of(profilerOutputLocation);
Expand Down
27 changes: 10 additions & 17 deletions hdt-java-core/src/main/java/org/rdfhdt/hdt/hdt/impl/HDTImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,11 +187,11 @@ public void mapFromHDT(File f, long offset, ProgressListener listener) throws IO
f = new File(hdtFileName);

if(!f.exists()) {
System.err.println("We cannot map a gzipped HDT, decompressing into "+hdtFileName+" first.");
log.warn("We cannot map a gzipped HDT, decompressing into {} first.", hdtFileName);
IOUtil.decompressGzip(old, f);
System.err.println("Gzipped HDT successfully decompressed. You might want to delete "+old.getAbsolutePath()+" to save disk space.");
log.warn("Gzipped HDT successfully decompressed. You might want to delete {} to save disk space.", old.getAbsolutePath());
} else {
System.err.println("We cannot map a gzipped HDT, using "+hdtFileName+" instead.");
log.warn("We cannot map a gzipped HDT, using {} instead.", hdtFileName);
}
}

Expand Down Expand Up @@ -345,20 +345,16 @@ public void loadFromModifiableHDT(TempHDT modHdt, ProgressListener listener) {

// Convert triples to final format
if(triples.getClass().equals(modifiableTriples.getClass())) {
triples = modifiableTriples;
triples = modifiableTriples;
} else {
//StopWatch tripleConvTime = new StopWatch();
triples.load(modifiableTriples, listener);
//System.out.println("Triples conversion time: "+tripleConvTime.stopAndShow());
triples.load(modifiableTriples, listener);
}

// Convert dictionary to final format
if(dictionary.getClass().equals(modifiableDictionary.getClass())) {
dictionary = (DictionaryPrivate)modifiableDictionary;
dictionary = (DictionaryPrivate)modifiableDictionary;
} else {
//StopWatch dictConvTime = new StopWatch();
dictionary.load(modifiableDictionary, listener);
//System.out.println("Dictionary conversion time: "+dictConvTime.stopAndShow());
dictionary.load(modifiableDictionary, listener);
}

this.baseUri = modHdt.getBaseURI();
Expand Down Expand Up @@ -396,8 +392,7 @@ public void loadOrCreateIndex(ProgressListener listener, HDTOptions spec) throws
}
} catch (Exception e) {
if (!(e instanceof FileNotFoundException)) {
System.out.println("Error reading .hdt.index, generating a new one. The error was: "+e.getMessage());
e.printStackTrace();
log.error("Error reading .hdt.index, generating a new one.", e);
}

// GENERATE
Expand All @@ -411,11 +406,9 @@ public void loadOrCreateIndex(ProgressListener listener, HDTOptions spec) throws
out = new BufferedOutputStream(new FileOutputStream(versionName));
ci.clear();
triples.saveIndex(out, ci, listener);
out.close();
ate47 marked this conversation as resolved.
Show resolved Hide resolved
System.out.println("Index generated and saved in "+st.stopAndShow());
log.info("Index generated and saved in {}", st.stopAndShow());
} catch (IOException e2) {
System.err.println("Error writing index file.");
e2.printStackTrace();
log.error("Error writing index file.", e2);
} finally {
IOUtil.closeQuietly(out);
}
Expand Down
7 changes: 2 additions & 5 deletions hdt-java-core/src/main/java/org/rdfhdt/hdt/rdf/parsers/RDFParserSimple.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,10 +72,7 @@ public void doParse(InputStream input, String baseUri, RDFNotation notation, boo
try {
doParse(reader, baseUri, notation, keepBNode, callback);
} finally {
try {
reader.close();
} catch (IOException e) {
}
IOUtil.closeQuietly(reader);
}
}

Expand Down Expand Up @@ -110,7 +107,7 @@ private void doParse(BufferedReader reader, String baseUri, RDFNotation notation
//System.out.println(triple);
callback.processTriple(triple, 0);
} else {
System.err.println("Warning: Could not parse triple at line "+numLine+", ignored and not processed.\n"+line);
log.warn("Could not parse triple at line {}, ignored and not processed.\n{}", numLine, line);
}
}
numLine++;
Expand Down
20 changes: 8 additions & 12 deletions hdt-java-core/src/main/java/org/rdfhdt/hdt/triples/impl/BitmapTriples.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -850,8 +850,8 @@ class Pair {
}

ArrayList<List<Pair>> list=new ArrayList<>();
System.out.println("Generating HDT Index for ?PO, and ??O queries.");

log.info("Generating HDT Index for ?PO, and ??O queries.");
// Generate lists
long total=seqZ.getNumberOfElements();
for(long i=0;i<total;i++) {
Expand All @@ -876,12 +876,12 @@ class Pair {

inner.add(pair);

if((i%100000)==0) {
System.out.println("Processed: "+i+" objects out of "+total);
if((i % 100000)==0) {
log.info("Processed {}/{} objects", i, total);
}
}
System.out.println("Serialize object lists");

log.info("Serialize object lists");
// Serialize
DynamicSequence indexZ = new SequenceLog64(BitUtil.log2(seqY.getNumberOfElements()), list.size());
Bitmap375Big bitmapIndexZ = Bitmap375Big.memory(seqY.getNumberOfElements());
Expand All @@ -903,7 +903,7 @@ class Pair {
}

if((i%100000)==0) {
System.out.println("Serialized: "+i+" lists out of "+total);
log.info("Serialized {}/{} lists", i, total);
}

// Dereference processed list to let GC release the memory.
Expand Down Expand Up @@ -955,11 +955,7 @@ public void generateIndex(ProgressListener listener, HDTOptions specIndex, Dicti
}

predicateIndex = new PredicateIndexArray(this);
if (!specIndex.getBoolean("debug.bitmaptriples.ignorePredicateIndex", false)) {
predicateIndex.generate(listener, specIndex, dictionary);
} else {
System.err.println("WARNING!!! PREDICATE INDEX IGNORED, THE INDEX WON'T BE COMPLETED!");
}
predicateIndex.generate(listener, specIndex, dictionary);
}

/* (non-Javadoc)
Expand Down
16 changes: 7 additions & 9 deletions hdt-java-core/src/main/java/org/rdfhdt/hdt/triples/impl/PredicateIndexArray.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,8 +102,8 @@ public void generate(ProgressListener listener, HDTOptions specIndex, Dictionary
maxCount = Math.max(count, maxCount);
predCount.set(val - 1, count);

if (listener != null && i % 1_000_000 == 0) {
listener.notifyProgress((float) (i * 100 / triples.getSeqY().getNumberOfElements()), "Counting appearances of predicates " + i + " / "+ triples.getSeqY().getNumberOfElements());
if (i % 1_000_000 == 0) {
iListener.notifyProgress((float) (i * 100 / triples.getSeqY().getNumberOfElements()), "Counting appearances of predicates " + i + " / "+ triples.getSeqY().getNumberOfElements());
}
}
predCount.aggressiveTrimToSize();
Expand All @@ -114,15 +114,13 @@ public void generate(ProgressListener listener, HDTOptions specIndex, Dictionary
for (long i = 0; i < predCount.getNumberOfElements(); i++) {
tempCountPred += predCount.get(i);
bitmap.set(tempCountPred - 1, true);
if (listener != null && i % 1_000_000 == 0) {
listener.notifyProgress((float) (i * 100 / predCount.getNumberOfElements()), "Creating Predicate bitmap " + i + " / "+ triples.getSeqY().getNumberOfElements());
if (i % 1_000_000 == 0) {
iListener.notifyProgress((float) (i * 100 / predCount.getNumberOfElements()), "Creating Predicate bitmap " + i + " / "+ triples.getSeqY().getNumberOfElements());
}
}
bitmap.set(triples.getSeqY().getNumberOfElements() - 1, true);
log.info("Predicate Bitmap in {}", st.stopAndShow());
if (listener != null) {
listener.notifyProgress(100, "Predicate Bitmap in " + st);
}
iListener.notifyProgress(100, "Predicate Bitmap in " + st);
st.reset();
} finally {
IOUtil.closeQuietly(predCount);
Expand All @@ -146,8 +144,8 @@ public void generate(ProgressListener listener, HDTOptions specIndex, Dictionary

array.set(insertBase + insertOffset, i);

if (listener != null && i % 1_000_000 == 0) {
listener.notifyProgress( (float) (i * 100 / triples.getSeqY().getNumberOfElements()), "Generating predicate references");
if (i % 1_000_000 == 0) {
iListener.notifyProgress( (float) (i * 100 / triples.getSeqY().getNumberOfElements()), "Generating predicate references");
}
}
} finally {
Expand Down