Skip to content

Commit

Permalink
Merge pull request #190 from ate47/fix_sout
Browse files Browse the repository at this point in the history 
Use logger instead of stdout/stderr outside of the the CLI
  • Loading branch information
@D063520
D063520 authored Feb 1, 2023
2 parents bf26d0a + ca770d9 commit 4e823ec
Show file tree
Hide file tree
Showing 5 changed files with 28 additions and 44 deletions.
2 changes: 1 addition & 1 deletion hdt-api/src/main/java/org/rdfhdt/hdt/util/Profiler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ public Profiler(String name, HDTOptions spec, boolean async) {
this.name = Objects.requireNonNull(name, "name can't be null!");
if (spec != null) {
String b = spec.get(async ? HDTOptionsKeys.PROFILER_ASYNC_KEY : HDTOptionsKeys.PROFILER_KEY);
disabled = b == null || b.length() == 0 || !(b.charAt(0) == '!' || "true".equalsIgnoreCase(b));
disabled = b == null || b.length() == 0 || !("true".equalsIgnoreCase(b));
String profilerOutputLocation = spec.get(async ? HDTOptionsKeys.PROFILER_ASYNC_OUTPUT_KEY : HDTOptionsKeys.PROFILER_OUTPUT_KEY);
if (profilerOutputLocation != null && !profilerOutputLocation.isEmpty()) {
outputPath = Path.of(profilerOutputLocation);
Expand Down
27 changes: 10 additions & 17 deletions hdt-java-core/src/main/java/org/rdfhdt/hdt/hdt/impl/HDTImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,11 +187,11 @@ public void mapFromHDT(File f, long offset, ProgressListener listener) throws IO
f = new File(hdtFileName);

if(!f.exists()) {
System.err.println("We cannot map a gzipped HDT, decompressing into "+hdtFileName+" first.");
log.warn("We cannot map a gzipped HDT, decompressing into {} first.", hdtFileName);
IOUtil.decompressGzip(old, f);
System.err.println("Gzipped HDT successfully decompressed. You might want to delete "+old.getAbsolutePath()+" to save disk space.");
log.warn("Gzipped HDT successfully decompressed. You might want to delete {} to save disk space.", old.getAbsolutePath());
} else {
System.err.println("We cannot map a gzipped HDT, using "+hdtFileName+" instead.");
log.warn("We cannot map a gzipped HDT, using {} instead.", hdtFileName);
}
}

Expand Down Expand Up @@ -345,20 +345,16 @@ public void loadFromModifiableHDT(TempHDT modHdt, ProgressListener listener) {

// Convert triples to final format
if(triples.getClass().equals(modifiableTriples.getClass())) {
triples = modifiableTriples;
triples = modifiableTriples;
} else {
//StopWatch tripleConvTime = new StopWatch();
triples.load(modifiableTriples, listener);
//System.out.println("Triples conversion time: "+tripleConvTime.stopAndShow());
triples.load(modifiableTriples, listener);
}

// Convert dictionary to final format
if(dictionary.getClass().equals(modifiableDictionary.getClass())) {
dictionary = (DictionaryPrivate)modifiableDictionary;
dictionary = (DictionaryPrivate)modifiableDictionary;
} else {
//StopWatch dictConvTime = new StopWatch();
dictionary.load(modifiableDictionary, listener);
//System.out.println("Dictionary conversion time: "+dictConvTime.stopAndShow());
dictionary.load(modifiableDictionary, listener);
}

this.baseUri = modHdt.getBaseURI();
Expand Down Expand Up @@ -396,8 +392,7 @@ public void loadOrCreateIndex(ProgressListener listener, HDTOptions spec) throws
}
} catch (Exception e) {
if (!(e instanceof FileNotFoundException)) {
System.out.println("Error reading .hdt.index, generating a new one. The error was: "+e.getMessage());
e.printStackTrace();
log.error("Error reading .hdt.index, generating a new one.", e);
}

// GENERATE
Expand All @@ -411,11 +406,9 @@ public void loadOrCreateIndex(ProgressListener listener, HDTOptions spec) throws
out = new BufferedOutputStream(new FileOutputStream(versionName));
ci.clear();
triples.saveIndex(out, ci, listener);
out.close();
System.out.println("Index generated and saved in "+st.stopAndShow());
log.info("Index generated and saved in {}", st.stopAndShow());
} catch (IOException e2) {
System.err.println("Error writing index file.");
e2.printStackTrace();
log.error("Error writing index file.", e2);
} finally {
IOUtil.closeQuietly(out);
}
Expand Down
7 changes: 2 additions & 5 deletions hdt-java-core/src/main/java/org/rdfhdt/hdt/rdf/parsers/RDFParserSimple.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,10 +72,7 @@ public void doParse(InputStream input, String baseUri, RDFNotation notation, boo
try {
doParse(reader, baseUri, notation, keepBNode, callback);
} finally {
try {
reader.close();
} catch (IOException e) {
}
IOUtil.closeQuietly(reader);
}
}

Expand Down Expand Up @@ -110,7 +107,7 @@ private void doParse(BufferedReader reader, String baseUri, RDFNotation notation
//System.out.println(triple);
callback.processTriple(triple, 0);
} else {
System.err.println("Warning: Could not parse triple at line "+numLine+", ignored and not processed.\n"+line);
log.warn("Could not parse triple at line {}, ignored and not processed.\n{}", numLine, line);
}
}
numLine++;
Expand Down
20 changes: 8 additions & 12 deletions hdt-java-core/src/main/java/org/rdfhdt/hdt/triples/impl/BitmapTriples.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -850,8 +850,8 @@ class Pair {
}

ArrayList<List<Pair>> list=new ArrayList<>();
System.out.println("Generating HDT Index for ?PO, and ??O queries.");

log.info("Generating HDT Index for ?PO, and ??O queries.");
// Generate lists
long total=seqZ.getNumberOfElements();
for(long i=0;i<total;i++) {
Expand All @@ -876,12 +876,12 @@ class Pair {

inner.add(pair);

if((i%100000)==0) {
System.out.println("Processed: "+i+" objects out of "+total);
if((i % 100000)==0) {
log.info("Processed {}/{} objects", i, total);
}
}
System.out.println("Serialize object lists");

log.info("Serialize object lists");
// Serialize
DynamicSequence indexZ = new SequenceLog64(BitUtil.log2(seqY.getNumberOfElements()), list.size());
Bitmap375Big bitmapIndexZ = Bitmap375Big.memory(seqY.getNumberOfElements());
Expand All @@ -903,7 +903,7 @@ class Pair {
}

if((i%100000)==0) {
System.out.println("Serialized: "+i+" lists out of "+total);
log.info("Serialized {}/{} lists", i, total);
}

// Dereference processed list to let GC release the memory.
Expand Down Expand Up @@ -955,11 +955,7 @@ public void generateIndex(ProgressListener listener, HDTOptions specIndex, Dicti
}

predicateIndex = new PredicateIndexArray(this);
if (!specIndex.getBoolean("debug.bitmaptriples.ignorePredicateIndex", false)) {
predicateIndex.generate(listener, specIndex, dictionary);
} else {
System.err.println("WARNING!!! PREDICATE INDEX IGNORED, THE INDEX WON'T BE COMPLETED!");
}
predicateIndex.generate(listener, specIndex, dictionary);
}

/* (non-Javadoc)
Expand Down
16 changes: 7 additions & 9 deletions hdt-java-core/src/main/java/org/rdfhdt/hdt/triples/impl/PredicateIndexArray.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,8 +102,8 @@ public void generate(ProgressListener listener, HDTOptions specIndex, Dictionary
maxCount = Math.max(count, maxCount);
predCount.set(val - 1, count);

if (listener != null && i % 1_000_000 == 0) {
listener.notifyProgress((float) (i * 100 / triples.getSeqY().getNumberOfElements()), "Counting appearances of predicates " + i + " / "+ triples.getSeqY().getNumberOfElements());
if (i % 1_000_000 == 0) {
iListener.notifyProgress((float) (i * 100 / triples.getSeqY().getNumberOfElements()), "Counting appearances of predicates " + i + " / "+ triples.getSeqY().getNumberOfElements());
}
}
predCount.aggressiveTrimToSize();
Expand All @@ -114,15 +114,13 @@ public void generate(ProgressListener listener, HDTOptions specIndex, Dictionary
for (long i = 0; i < predCount.getNumberOfElements(); i++) {
tempCountPred += predCount.get(i);
bitmap.set(tempCountPred - 1, true);
if (listener != null && i % 1_000_000 == 0) {
listener.notifyProgress((float) (i * 100 / predCount.getNumberOfElements()), "Creating Predicate bitmap " + i + " / "+ triples.getSeqY().getNumberOfElements());
if (i % 1_000_000 == 0) {
iListener.notifyProgress((float) (i * 100 / predCount.getNumberOfElements()), "Creating Predicate bitmap " + i + " / "+ triples.getSeqY().getNumberOfElements());
}
}
bitmap.set(triples.getSeqY().getNumberOfElements() - 1, true);
log.info("Predicate Bitmap in {}", st.stopAndShow());
if (listener != null) {
listener.notifyProgress(100, "Predicate Bitmap in " + st);
}
iListener.notifyProgress(100, "Predicate Bitmap in " + st);
st.reset();
} finally {
IOUtil.closeQuietly(predCount);
Expand All @@ -146,8 +144,8 @@ public void generate(ProgressListener listener, HDTOptions specIndex, Dictionary

array.set(insertBase + insertOffset, i);

if (listener != null && i % 1_000_000 == 0) {
listener.notifyProgress( (float) (i * 100 / triples.getSeqY().getNumberOfElements()), "Generating predicate references");
if (i % 1_000_000 == 0) {
iListener.notifyProgress( (float) (i * 100 / triples.getSeqY().getNumberOfElements()), "Generating predicate references");
}
}
} finally {
Expand Down

0 comments on commit 4e823ec

Please sign in to comment.