Skip to content

Commit

Permalink
Merge pull request #63 from salesforce/jdk-17
Browse files Browse the repository at this point in the history 
fixed locale changes by JDK 17
  • Loading branch information
@yoikawa
yoikawa authored Dec 20, 2024
2 parents 840b447 + 84943ac commit 54b79e8
Show file tree
Hide file tree
Showing 27 changed files with 511 additions and 465 deletions.
83 changes: 50 additions & 33 deletions src/main/java/com/force/i18n/LabelUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ public static String getSampleGrammarFile(@Nullable String grammar) {
} else {
// If they pasted in a whole file, it's fine.
if (!grammar.contains("<sfdcnames>") && !grammar.contains("<sfdcadjectives>")
&& !grammar.contains("<names>") && !grammar.contains("<adjectives>")) {
&& !grammar.contains("<names>") && !grammar.contains("<adjectives>")) {
StringBuilder sb = new StringBuilder(grammar.length() + 100);
sb.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
sb.append("<names>");
Expand All @@ -139,7 +139,7 @@ public static String getSampleGrammarFile(@Nullable String grammar) {
return sb.toString();
} else {
// Remove the dtd which confuses the parser
String fixedGrammar = grammar.replace("<!DOCTYPE sfdcnames SYSTEM \"sfdcnames.dtd\">", "");
String fixedGrammar = grammar.replace("<!DOCTYPE sfdcnames SYSTEM \"sfdcnames.dtd\">", "");
fixedGrammar = fixedGrammar.replace("<!DOCTYPE sfdcadjectives SYSTEM \"sfdcadjectives.dtd\">", "");
fixedGrammar = fixedGrammar.replace("<!DOCTYPE names SYSTEM \"names.dtd\">", "");
fixedGrammar = fixedGrammar.replace("<!DOCTYPE adjectives SYSTEM \"adjectives.dtd\">", "");
Expand Down Expand Up @@ -180,46 +180,63 @@ public String nounify(String input, LanguageDictionary dictionary) {
return new Nounifier(dictionary).nounifyString(input);
}

/**
* Set of locale langauges strings where the results are JDK dependent
/*
* Map of locale langauges strings where the results are JDK dependent.
* Java 17 fixed legacy language code and autmatically replace it in {@code java.util.Locale} (JDK-8267069). This is
* for backward compatibility.
*/
static final Set<String> JDK_DEPENDENT_LANGUAGE = ImmutableSet.of(LanguageConstants.YIDDISH_ISO,
LanguageConstants.HEBREW_ISO, LanguageConstants.INDONESIAN_ISO);

static final Map<String, String> JDK_DEPENDENT_LANGUAGE = Map.of( //
LanguageConstants.YIDDISH_ISO, LanguageConstants.YIDDISH, //
LanguageConstants.HEBREW_ISO, LanguageConstants.HEBREW, //
LanguageConstants.INDONESIAN_ISO, LanguageConstants.INDONESIAN);

public static List<URL> getFileNames(HumanLanguage language,URL rootDirectory, String basename ) {
public static List<URL> getFileNames(HumanLanguage language, URL rootDirectory, String basename ) {
List<URL> list = new ArrayList<>();
try {
Locale locale = language.getLocale();
list.add(new URL(rootDirectory, locale.getLanguage() + '/' + basename));
final Locale locale = language.getLocale();
final String override = language.getOverrideLanguage();
final boolean hasOverride = override != null && !locale.getLanguage().equals(override);

list.add(getUrl(rootDirectory, locale.getLanguage(), basename));
// If the override language is set, add it
if (hasOverride) list.add(getUrl(rootDirectory, override, basename));
// If the iso code for the language changed between JDK versions, use the old isocode for the directory
if (JDK_DEPENDENT_LANGUAGE.contains(locale.getLanguage())) {
list.add(new URL(rootDirectory, language.getOverrideLanguage() + '/' + basename));
if (JDK_DEPENDENT_LANGUAGE.containsKey(locale.getLanguage())) {
list.add(getUrl(rootDirectory, JDK_DEPENDENT_LANGUAGE.get(locale.getLanguage()), basename));
}

if (!locale.getCountry().isEmpty()) {
/*
* Code required because there is no Chinese traditional locale in jdk Locale.java and taiwan and Hong
* kong are 2 different countries
* We will assume that Hong Kong (HK country code) derive the Traditional Chinese from Taiwan (TW)
* In the future, to define Hong Kong label, a new directory will need to be created zh/HK.
*/
if (language.getLocaleString().equals(LanguageConstants.CHINESE_HK)) {
// adding Taiwan as fallback. assuming no override, or variant are set for this
list.add(getUrl(rootDirectory, locale.getLanguage(), Locale.TRADITIONAL_CHINESE.getCountry(), basename));
list.add(getUrl(rootDirectory, locale.getLanguage(), locale.getCountry(), basename));
} else {
list.add(getUrl(rootDirectory, locale.getLanguage(), locale.getCountry(), basename));
if (hasOverride) list.add(getUrl(rootDirectory, override, locale.getCountry(), basename));

if (!locale.getVariant().isEmpty()) {
list.add(getUrl(rootDirectory, locale.getLanguage(), locale.getCountry(), locale.getVariant(), basename));
if (hasOverride) list.add(getUrl(rootDirectory, override, locale.getCountry(), locale.getVariant(), basename));
}
}
}
if (locale.getCountry().length() > 0) {
/*
* Code required because there is no Chinese traditional locale in jdk Locale.java and taiwan and Hong kong are 2 different countries
* We will assume that Hong Kong (HK country code) derive the Traditional Chinese from Taiwan (TW)
* In the future, to define Hong Kong label, a new directory will need to be created zh/HK.
*/
if(language.getLocaleString().equals(LanguageConstants.CHINESE_HK)){
//adding Taiwan as fallback
list.add(new URL(rootDirectory, locale.getLanguage() + '/'+ Locale.TRADITIONAL_CHINESE.getCountry() +'/'+ basename));
list.add(new URL(rootDirectory, locale.getLanguage() + '/'+ locale.getCountry() + '/'+ basename));
}else{
list.add(new URL(rootDirectory, locale.getLanguage() + '/' + locale.getCountry() + '/' + basename));
if (locale.getVariant().length() > 0) {
list.add(new URL(rootDirectory, locale.getLanguage() + '/' + locale.getCountry() + '/' + locale.getVariant() + '/' + basename));
}
}
}
} catch (MalformedURLException e) {
throw new IllegalArgumentException(e);
}
return list;
}
return list;
}

private static URL getUrl(URL root, String... paths) throws MalformedURLException {
return new URL(root, String.join("/", paths));
}

private static final Pattern nonalphaPattern = Pattern.compile("\\W");
private static final Pattern nonalphaPattern = Pattern.compile("\\W");

// TODO: Switch this to splitter.
static List<String> tokenize(String input) {
Expand All @@ -244,7 +261,7 @@ static List<String> tokenize(String input) {
* to determine if there should be some "tokenization" of the string.
*/
public static class Nounifier {
private final static Set<String> EXCLUDED_NOUNS = ImmutableSet.of("Role", "Email", "Address", "{0}");
private static final Set<String> EXCLUDED_NOUNS = ImmutableSet.of("Role", "Email", "Address", "{0}");

private final GenericTrieMatcher<String> nounMatcher;
private final GenericTrieMatcher<String> adjMatcher;
Expand Down
35 changes: 24 additions & 11 deletions src/main/java/com/force/i18n/commons/util/collection/IntHashMap.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
package com.force.i18n.commons.util.collection;

import java.io.IOException;
import java.io.Serializable;
import java.util.*;

import com.google.common.annotations.Beta;
Expand Down Expand Up @@ -77,12 +76,12 @@
* Beta class. Generally use {@link java.util.HashMap} or {@link java.util.EnumMap} instead.
*
* @author Based on Sun's java.util.HashMap (modified by koliver)
* @see IntMap
* @see java.util.HashMap
* @see IntMap
* @see java.util.HashMap
*/
@Beta
@SuppressWarnings("rawtypes") // TODO Fix
public class IntHashMap<V> extends AbstractIntMap<V> implements Serializable {
public class IntHashMap<V> extends AbstractIntMap<V> {
private static final long serialVersionUID = 0L;

private static final int DEFAULT_CAPACITY = 101;
Expand Down Expand Up @@ -186,7 +185,7 @@ public int size() {
*
* @param value value whose presence in this map is to be tested.
*/
@Override
@Override
public boolean containsValue(Object value) {
IEntry[] tab = this.table;

Expand Down Expand Up @@ -287,9 +286,9 @@ private void rehash() {
* @param key key with which the specified value is to be associated.
* @param value value to be associated with the specified key.
* @return previous value associated with specified key, or {@code null}
* if there was no mapping for key. A {@code null} return can
* also indicate that the HashMap previously associated
* {@code null} with the specified key.
* if there was no mapping for key. A {@code null} return can
* also indicate that the HashMap previously associated
* {@code null} with the specified key.
*/
@Override
public V put(int key, V value) {
Expand Down Expand Up @@ -328,9 +327,9 @@ public V put(int key, V value) {
*
* @param key key whose mapping is to be removed from the map.
* @return previous value associated with specified key, or {@code null}
* if there was no mapping for key. A {@code null} return can
* also indicate that the map previously associated {@code null}
* with the specified key.
* if there was no mapping for key. A {@code null} return can
* also indicate that the map previously associated {@code null}
* with the specified key.
*/
@Override
public V remove(int key) {
Expand Down Expand Up @@ -734,6 +733,20 @@ public int next() {
}
}

@Override
public int hashCode() {
return super.hashCode();
}

@Override
public boolean equals(Object other) {
if (other instanceof IntHashMap) {
// super compares table and count. that may be enough
return super.equals(other);
}
return false;
}

// Serialization
private void writeObject(java.io.ObjectOutputStream s) throws IOException {
// Write out the threshold, loadfactor, and any hidden stuff
Expand Down
31 changes: 15 additions & 16 deletions src/main/java/com/force/i18n/grammar/LanguageDictionary.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ public class LanguageDictionary implements Serializable {
// TODO: These could all be made lists when serialized

// map to Noun
protected GrammaticalTermMap<Noun> nounMap;
protected GrammaticalTermMap<Noun> nounMap;
// map to Noun
protected GrammaticalTermMap<Noun> nounMapByPluralAlias;
// map to Adjective
Expand Down Expand Up @@ -202,9 +202,9 @@ public GrammaticalTermMap<Noun> getNounByPluralAlias() {
public GrammaticalTermMap<Adjective> getAdjectiveMap() {
return adjectiveMap;
}

/**
* get termMap of Article
* get termMap of Article
*/
public GrammaticalTermMap<Article> getArticleMap() {
return articleMap;
Expand All @@ -213,10 +213,10 @@ public GrammaticalTermMap<Article> getArticleMap() {

private void forAllTerms (TermType type, BiConsumer<String, GrammaticalTerm> f) {
switch (type) {
case Noun:
case Noun:
for(Map.Entry<String, Noun> e : nounMap.entrySet()) f.accept(e.getKey(), e.getValue());
break;
case Adjective:
case Adjective:
for(Map.Entry<String, Adjective> e : adjectiveMap.entrySet()) f.accept(e.getKey(), e.getValue());
break;
case Article:
Expand All @@ -226,14 +226,14 @@ private void forAllTerms (TermType type, BiConsumer<String, GrammaticalTerm> f)
throw new AssertionError("Invalid term type " + type);
}
}



public Set<String> getAllTermNames(TermType type) {
switch (type) {
case Noun:
case Noun:
return Collections.unmodifiableSet(nounMap.keySet());
case Adjective:
case Adjective:
return Collections.unmodifiableSet(adjectiveMap.keySet());
case Article:
return Collections.unmodifiableSet(articleMap.keySet());
Expand All @@ -249,8 +249,8 @@ public Set<String> getAllTermNames(TermType type) {
public Set<String> getAllInheritedTermNames(TermType type) {
Set<String> result = new HashSet<String>();
forAllTerms(type, (k,v) -> {
if (v.isCopiedFromDefault()) result.add(k);
});
if (v.isCopiedFromDefault()) result.add(k);
});
return result;
}

Expand Down Expand Up @@ -669,12 +669,12 @@ public void writeJson(Appendable appendable, boolean useRenamedNouns, Collection
* @throws IOException if an error happens during append
*/
public void writeJsonTerms(Appendable out, boolean useRenamedNouns, Collection<GrammaticalTerm> terms) throws IOException {
Collection<String> termsToInclude =
terms == null ? null :
Collection<String> termsToInclude =
terms == null ? null :
terms.stream().map((t)->t.getName()).collect(Collectors.toSet());
writeJson(out, useRenamedNouns, termsToInclude);
}


private void writeObject(ObjectOutputStream out) throws IOException {
makeSkinny();
Expand All @@ -686,7 +686,6 @@ protected void writeObjectInternal(ObjectOutputStream out) throws IOException {
// do nothing here - for hook
}

@SuppressWarnings("unchecked")
private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
in.defaultReadObject();

Expand Down Expand Up @@ -720,9 +719,9 @@ public Noun getNounOverride(Noun n) {
}

private static class TreeSetSupplier<V> implements Supplier<SortedSet<V>>, Serializable {
private static final long serialVersionUID = 1L;
private static final long serialVersionUID = 1L;

@Override
@Override
public SortedSet<V> get() {
return new TreeSet<V>();
}
Expand Down
8 changes: 5 additions & 3 deletions src/main/java/com/force/i18n/grammar/impl/ComplexGrammaticalForm.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
import com.force.i18n.grammar.ModifierForm;
import com.force.i18n.grammar.Noun;
import com.force.i18n.grammar.NounForm;
import com.google.common.base.Preconditions;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Multimap;
Expand Down Expand Up @@ -137,7 +138,7 @@ protected Object readResolve() {
case Adjective: formList = declension.getAdjectiveForms(); break;
case Article: formList = declension.getArticleForms(); break;
}
assert formList != null;
Preconditions.checkState(formList != null);
return formList.get(this.ordinal);
}
}
Expand Down Expand Up @@ -273,7 +274,7 @@ static <T extends ComplexGrammaticalForm> void serializeFormMap(ObjectOutputStre
} else {
out.writeByte(values.size());
for (Map.Entry<T,String> entry : values.entrySet()) {
out.writeByte(entry.getKey().getOrdinal());
out.writeByte(entry.getKey().getOrdinal());
out.writeUTF(entry.getValue()); // Serialize the "object" because it's been uniquefied
}
}
Expand All @@ -292,7 +293,8 @@ static <T extends ComplexGrammaticalForm> Map<T, String> deserializeFormMap(Obje
case Adjective: formList = (List<T>)declension.getAdjectiveForms(); break;
case Article: formList = (List<T>)declension.getArticleForms(); break;
}
assert formList != null;
Preconditions.checkState(formList != null);

for (int i = 0; i < size; i++) {
int ordinal = in.readByte();
String value = intern(in.readUTF());
Expand Down
35 changes: 18 additions & 17 deletions src/main/java/com/force/i18n/grammar/impl/EnglishDeclension.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
package com.force.i18n.grammar.impl;

import static com.force.i18n.commons.util.settings.IniFileUtil.intern;
import static com.google.common.base.Preconditions.checkArgument;

import java.io.IOException;
import java.util.*;
Expand All @@ -25,18 +26,18 @@
* @author stamm
*/
class EnglishDeclension extends ArticledDeclension {
private static final Logger logger = Logger.getLogger(EnglishDeclension.class.getName());
private static final Logger logger = Logger.getLogger(EnglishDeclension.class.getName());

public EnglishDeclension(HumanLanguage language) {
super(language);
assert language.getLocale().getLanguage().equals("en") : "Initializing a language that isn't english";
}
public EnglishDeclension(HumanLanguage language) {
super(language);
checkArgument(language.getLocale().getLanguage().equals("en"), "Initializing a language that isn't english");
}

/**
* The english articles are distinguished by whether the next noun starts with a vowel
* sound or not (although the, unlike a/an is spelled the same).
*/
public static enum EnglishArticleForm implements ArticleForm {
public enum EnglishArticleForm implements ArticleForm {
SINGULAR(LanguageNumber.SINGULAR, LanguageStartsWith.CONSONANT),
SINGULAR_V(LanguageNumber.SINGULAR, LanguageStartsWith.VOWEL),
PLURAL(LanguageNumber.PLURAL, LanguageStartsWith.CONSONANT)
Expand All @@ -59,17 +60,17 @@ static EnglishArticleForm getForm(ModifierForm form) {
(form.getStartsWith() == LanguageStartsWith.VOWEL ? SINGULAR_V : SINGULAR)
: PLURAL;
}
@Override
public String getKey() {
return getNumber().getDbValue() + "-" + getStartsWith().getDbValue();
}

@Override
public void appendJsFormReplacement(Appendable a, String termFormVar, String genderVar, String startsWithVar)
throws IOException {
// The only variant is in the signular, so we don't want to screw up "the"
a.append(termFormVar+".charAt(0)=='"+LanguageNumber.PLURAL.getDbValue()+"'?"+termFormVar+":'"+LanguageNumber.SINGULAR.getDbValue()+"-'+"+startsWithVar);
}
@Override
public String getKey() {
return getNumber().getDbValue() + "-" + getStartsWith().getDbValue();
}

@Override
public void appendJsFormReplacement(Appendable a, String termFormVar, String genderVar, String startsWithVar)
throws IOException {
// The only variant is in the signular, so we don't want to screw up "the"
a.append(termFormVar+".charAt(0)=='"+LanguageNumber.PLURAL.getDbValue()+"'?"+termFormVar+":'"+LanguageNumber.SINGULAR.getDbValue()+"-'+"+startsWithVar);
}
}

/**
Expand Down
Loading

0 comments on commit 54b79e8

Please sign in to comment.