From 939957124a8f055c7d343d67d0842ae06cf59530 Mon Sep 17 00:00:00 2001 From: Thomas A Cellucci Date: Thu, 13 Jul 2017 13:54:13 -0700 Subject: [PATCH] memory tuning --- build.gradle | 4 +- eureka-client/build.gradle | 4 +- .../appinfo/AbstractInstanceConfig.java | 8 +- .../java/com/netflix/appinfo/AmazonInfo.java | 13 +- .../com/netflix/appinfo/InstanceInfo.java | 76 +- .../discovery/AbstractAzToRegionMapper.java | 2 +- .../netflix/discovery/DiscoveryClient.java | 20 +- .../discovery/TimedSupervisorTask.java | 3 +- .../discovery/converters/EnumLookup.java | 104 +++ .../converters/EurekaJacksonCodec.java | 670 +++++++++++------- .../StringInterningAmazonInfoBuilder.java | 68 +- .../netflix/discovery/shared/Application.java | 29 +- .../discovery/shared/Applications.java | 411 ++++------- .../util/DeserializerStringCache.java | 463 ++++++++++++ .../util/EurekaEntityComparators.java | 3 +- .../discovery/converters/EnumLookupTest.java | 26 + .../EurekaJacksonCodecIntegrationTest.java | 117 +++ .../converters/EurekaJacksonCodecTest.java | 13 +- .../discovery/shared/ApplicationsTest.java | 340 ++++++++- .../eureka/registry/RemoteRegionRegistry.java | 12 - 20 files changed, 1738 insertions(+), 648 deletions(-) create mode 100644 eureka-client/src/main/java/com/netflix/discovery/converters/EnumLookup.java create mode 100644 eureka-client/src/main/java/com/netflix/discovery/util/DeserializerStringCache.java create mode 100644 eureka-client/src/test/java/com/netflix/discovery/converters/EnumLookupTest.java create mode 100644 eureka-client/src/test/java/com/netflix/discovery/converters/EurekaJacksonCodecIntegrationTest.java diff --git a/build.gradle b/build.gradle index b966d28437..d67c6f3fa4 100644 --- a/build.gradle +++ b/build.gradle @@ -52,8 +52,8 @@ subprojects { group = "com.netflix.${githubProjectName}" - sourceCompatibility = 1.7 - targetCompatibility = 1.7 + sourceCompatibility = 1.8 + targetCompatibility = 1.8 repositories { jcenter() diff --git a/eureka-client/build.gradle b/eureka-client/build.gradle index adaee99c1c..6a886ca53b 100644 --- a/eureka-client/build.gradle +++ b/eureka-client/build.gradle @@ -1,5 +1,5 @@ apply plugin: 'nebula.test-jar' -apply plugin: 'provided-base' +apply plugin: 'nebula.provided-base' configurations.all { // jersey2 @@ -18,6 +18,8 @@ dependencies { compile "org.apache.httpcomponents:httpclient:${apacheHttpClientVersion}" compile "com.google.inject:guice:${guiceVersion}" + compile "com.github.vlsi.compactmap:compactmap:1.2.1" + compile "com.fasterxml.jackson.core:jackson-annotations:${jacksonVersion}" compile "com.fasterxml.jackson.core:jackson-core:${jacksonVersion}" compile "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}" diff --git a/eureka-client/src/main/java/com/netflix/appinfo/AbstractInstanceConfig.java b/eureka-client/src/main/java/com/netflix/appinfo/AbstractInstanceConfig.java index c32e73b14f..e723a321e7 100644 --- a/eureka-client/src/main/java/com/netflix/appinfo/AbstractInstanceConfig.java +++ b/eureka-client/src/main/java/com/netflix/appinfo/AbstractInstanceConfig.java @@ -212,13 +212,13 @@ public String getIpAddress() { } private static Pair getHostInfo() { - Pair pair = new Pair("", ""); + Pair pair; try { - pair.setFirst(InetAddress.getLocalHost().getHostAddress()); - pair.setSecond(InetAddress.getLocalHost().getHostName()); - + InetAddress localHost = InetAddress.getLocalHost(); + pair = new Pair(localHost.getHostAddress(), localHost.getHostName()); } catch (UnknownHostException e) { logger.error("Cannot get host info", e); + pair = new Pair("", ""); } return pair; } diff --git a/eureka-client/src/main/java/com/netflix/appinfo/AmazonInfo.java b/eureka-client/src/main/java/com/netflix/appinfo/AmazonInfo.java index 0299735b14..c4d81730f2 100644 --- a/eureka-client/src/main/java/com/netflix/appinfo/AmazonInfo.java +++ b/eureka-client/src/main/java/com/netflix/appinfo/AmazonInfo.java @@ -48,11 +48,9 @@ * @author Karthik Ranganathan, Greg Kim * */ -@JsonDeserialize(builder = StringInterningAmazonInfoBuilder.class) +@JsonDeserialize(using = StringInterningAmazonInfoBuilder.class) public class AmazonInfo implements DataCenterInfo, UniqueIdentifier { - private Map metadata = new HashMap(); - private static final String AWS_API_VERSION = "latest"; private static final String AWS_METADATA_URL = "http://169.254.169.254/" + AWS_API_VERSION + "/meta-data/"; @@ -239,7 +237,10 @@ public AmazonInfo autoBuild(String namespace) { } } + private Map metadata; + public AmazonInfo() { + this.metadata = new HashMap(); } /** @@ -254,6 +255,12 @@ public AmazonInfo( @JsonProperty("metadata") HashMap metadata) { this.metadata = metadata; } + + public AmazonInfo( + @JsonProperty("name") String name, + @JsonProperty("metadata") Map metadata) { + this.metadata = metadata; + } @Override public Name getName() { diff --git a/eureka-client/src/main/java/com/netflix/appinfo/InstanceInfo.java b/eureka-client/src/main/java/com/netflix/appinfo/InstanceInfo.java index c337b4a916..14f9c8dcb2 100644 --- a/eureka-client/src/main/java/com/netflix/appinfo/InstanceInfo.java +++ b/eureka-client/src/main/java/com/netflix/appinfo/InstanceInfo.java @@ -22,6 +22,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -56,6 +57,8 @@ @JsonRootName("instance") public class InstanceInfo { + private static final String VERSION_UNKNOWN = "unknown"; + /** * {@link InstanceInfo} JSON and XML format for port information does not follow the usual conventions, which * makes its mapping complicated. This class represents the wire format for port information. @@ -141,18 +144,21 @@ public int getPort() { @Auto private volatile Boolean isCoordinatingDiscoveryServer = Boolean.FALSE; @XStreamAlias("metadata") - private volatile Map metadata = new ConcurrentHashMap(); + private volatile Map metadata; @Auto - private volatile Long lastUpdatedTimestamp = System.currentTimeMillis(); + private volatile Long lastUpdatedTimestamp; @Auto - private volatile Long lastDirtyTimestamp = System.currentTimeMillis(); + private volatile Long lastDirtyTimestamp; @Auto private volatile ActionType actionType; @Auto private volatile String asgName; - private String version = "unknown"; + private String version = VERSION_UNKNOWN; private InstanceInfo() { + this.metadata = new ConcurrentHashMap(); + this.lastUpdatedTimestamp = System.currentTimeMillis(); + this.lastDirtyTimestamp = lastUpdatedTimestamp; } @JsonCreator @@ -308,9 +314,12 @@ public enum InstanceStatus { UNKNOWN; public static InstanceStatus toEnum(String s) { - for (InstanceStatus e : InstanceStatus.values()) { - if (e.name().equalsIgnoreCase(s)) { - return e; + if (s != null) { + try { + return InstanceStatus.valueOf(s.toUpperCase()); + } catch (IllegalArgumentException e) { + // ignore and fall through to unknown + if (logger.isDebugEnabled()) logger.debug("illegal argument supplied to InstanceStatus.valueOf: {}, defaulting to {}", s, UNKNOWN); } } return UNKNOWN; @@ -319,10 +328,8 @@ public static InstanceStatus toEnum(String s) { @Override public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((getId() == null) ? 0 : getId().hashCode()); - return result; + String id = getId(); + return (id == null) ? 31 : (id.hashCode() + 31); } @Override @@ -337,11 +344,12 @@ public boolean equals(Object obj) { return false; } InstanceInfo other = (InstanceInfo) obj; - if (getId() == null) { + String id = getId(); + if (id == null) { if (other.getId() != null) { return false; } - } else if (!getId().equals(other.getId())) { + } else if (!id.equals(other.getId())) { return false; } return true; @@ -355,6 +363,7 @@ public static final class Builder { private static final String COLON = ":"; private static final String HTTPS_PROTOCOL = "https://"; private static final String HTTP_PROTOCOL = "http://"; + private final Function intern; private static final class LazyHolder { private static final VipAddressResolver DEFAULT_VIP_ADDRESS_RESOLVER = new Archaius1VipAddressResolver(); @@ -368,21 +377,26 @@ private static final class LazyHolder { private String namespace; - private Builder(InstanceInfo result, VipAddressResolver vipAddressResolver) { + private Builder(InstanceInfo result, VipAddressResolver vipAddressResolver, Function intern) { this.vipAddressResolver = vipAddressResolver; this.result = result; + this.intern = intern != null ? intern : StringCache::intern; } public Builder(InstanceInfo instanceInfo) { - this(instanceInfo, LazyHolder.DEFAULT_VIP_ADDRESS_RESOLVER); + this(instanceInfo, LazyHolder.DEFAULT_VIP_ADDRESS_RESOLVER, null); } public static Builder newBuilder() { - return new Builder(new InstanceInfo(), LazyHolder.DEFAULT_VIP_ADDRESS_RESOLVER); + return new Builder(new InstanceInfo(), LazyHolder.DEFAULT_VIP_ADDRESS_RESOLVER, null); + } + + public static Builder newBuilder(Function intern) { + return new Builder(new InstanceInfo(), LazyHolder.DEFAULT_VIP_ADDRESS_RESOLVER, intern); } public static Builder newBuilder(VipAddressResolver vipAddressResolver) { - return new Builder(new InstanceInfo(), vipAddressResolver); + return new Builder(new InstanceInfo(), vipAddressResolver, null); } public Builder setInstanceId(String instanceId) { @@ -398,18 +412,28 @@ public Builder setInstanceId(String instanceId) { * @return the instance info builder. */ public Builder setAppName(String appName) { - result.appName = StringCache.intern(appName.toUpperCase(Locale.ROOT)); + result.appName = intern.apply(appName.toUpperCase(Locale.ROOT)); return this; } + + public Builder setAppNameForDeser(String appName) { + result.appName = appName; + return this; + } + public Builder setAppGroupName(String appGroupName) { if (appGroupName != null) { - result.appGroupName = appGroupName.toUpperCase(Locale.ROOT); + result.appGroupName = intern.apply(appGroupName.toUpperCase(Locale.ROOT)); } else { result.appGroupName = null; } return this; } + public Builder setAppGroupNameForDeser(String appGroupName) { + result.appGroupName = appGroupName; + return this; + } /** * Sets the fully qualified hostname of this running instance.This is @@ -675,8 +699,8 @@ public Builder setHealthCheckUrlsForDeser(String healthCheckUrl, String secureHe * @return the instance builder. */ public Builder setVIPAddress(final String vipAddress) { - result.vipAddressUnresolved = StringCache.intern(vipAddress); - result.vipAddress = StringCache.intern( + result.vipAddressUnresolved = intern.apply(vipAddress); + result.vipAddress = intern.apply( vipAddressResolver.resolveDeploymentContextBasedVipAddresses(vipAddress)); return this; } @@ -685,7 +709,7 @@ public Builder setVIPAddress(final String vipAddress) { * Setter used during deserialization process, that does not do macro expansion on the provided value. */ public Builder setVIPAddressDeser(String vipAddress) { - result.vipAddress = StringCache.intern(vipAddress); + result.vipAddress = intern.apply(vipAddress); return this; } @@ -699,8 +723,8 @@ public Builder setVIPAddressDeser(String vipAddress) { * @return - Builder instance */ public Builder setSecureVIPAddress(final String secureVIPAddress) { - result.secureVipAddressUnresolved = StringCache.intern(secureVIPAddress); - result.secureVipAddress = StringCache.intern( + result.secureVipAddressUnresolved = intern.apply(secureVIPAddress); + result.secureVipAddress = intern.apply( vipAddressResolver.resolveDeploymentContextBasedVipAddresses(secureVIPAddress)); return this; } @@ -709,7 +733,7 @@ public Builder setSecureVIPAddress(final String secureVIPAddress) { * Setter used during deserialization process, that does not do macro expansion on the provided value. */ public Builder setSecureVIPAddressDeser(String secureVIPAddress) { - result.secureVipAddress = StringCache.intern(secureVIPAddress); + result.secureVipAddress = intern.apply(secureVIPAddress); return this; } @@ -791,7 +815,7 @@ public boolean isInitialized() { * @return the instance info builder. */ public Builder setASGName(String asgName) { - result.asgName = StringCache.intern(asgName); + result.asgName = intern.apply(asgName); return this; } diff --git a/eureka-client/src/main/java/com/netflix/discovery/AbstractAzToRegionMapper.java b/eureka-client/src/main/java/com/netflix/discovery/AbstractAzToRegionMapper.java index 2e0795987e..0870f155b4 100644 --- a/eureka-client/src/main/java/com/netflix/discovery/AbstractAzToRegionMapper.java +++ b/eureka-client/src/main/java/com/netflix/discovery/AbstractAzToRegionMapper.java @@ -57,7 +57,7 @@ public synchronized void setRegionsToFetch(String[] regionsToFetch) { for (String remoteRegion : regionsToFetch) { Set availabilityZones = getZonesForARegion(remoteRegion); if (null == availabilityZones - || (availabilityZones.size() == 1 && availabilityZones.iterator().next().equals(DEFAULT_ZONE)) + || (availabilityZones.size() == 1 && availabilityZones.contains(DEFAULT_ZONE)) || availabilityZones.isEmpty()) { logger.info("No availability zone information available for remote region: " + remoteRegion + ". Now checking in the default mapping."); diff --git a/eureka-client/src/main/java/com/netflix/discovery/DiscoveryClient.java b/eureka-client/src/main/java/com/netflix/discovery/DiscoveryClient.java index 5657aedab4..fc19dc1500 100644 --- a/eureka-client/src/main/java/com/netflix/discovery/DiscoveryClient.java +++ b/eureka-client/src/main/java/com/netflix/discovery/DiscoveryClient.java @@ -768,7 +768,7 @@ public InstanceInfo getNextServerFromEureka(String virtualHostname, boolean secu + virtualHostname); } Applications apps = this.localRegionApps.get(); - int index = (int) (apps.getNextIndex(virtualHostname.toUpperCase(Locale.ROOT), + int index = (int) (apps.getNextIndex(virtualHostname, secure).incrementAndGet() % instanceInfoList.size()); return instanceInfoList.get(index); } @@ -1135,24 +1135,6 @@ private void reconcileAndLogDifference(Applications delta, String reconcileHashC return; } - if (logger.isDebugEnabled()) { - try { - Map> reconcileDiffMap = getApplications().getReconcileMapDiff(serverApps); - StringBuilder reconcileBuilder = new StringBuilder(""); - for (Map.Entry> mapEntry : reconcileDiffMap.entrySet()) { - reconcileBuilder.append(mapEntry.getKey()).append(": "); - for (String displayString : mapEntry.getValue()) { - reconcileBuilder.append(displayString); - } - reconcileBuilder.append('\n'); - } - String reconcileString = reconcileBuilder.toString(); - logger.debug("The reconcile string is {}", reconcileString); - } catch (Throwable e) { - logger.error("Could not calculate reconcile string ", e); - } - } - if (fetchRegistryGeneration.compareAndSet(currentUpdateGeneration, currentUpdateGeneration + 1)) { localRegionApps.set(this.filterAndShuffle(serverApps)); getApplications().setVersion(delta.getVersion()); diff --git a/eureka-client/src/main/java/com/netflix/discovery/TimedSupervisorTask.java b/eureka-client/src/main/java/com/netflix/discovery/TimedSupervisorTask.java index ad6f35776a..808458cd1e 100644 --- a/eureka-client/src/main/java/com/netflix/discovery/TimedSupervisorTask.java +++ b/eureka-client/src/main/java/com/netflix/discovery/TimedSupervisorTask.java @@ -55,8 +55,9 @@ public TimedSupervisorTask(String name, ScheduledExecutorService scheduler, Thre Monitors.registerObject(name, this); } + @Override public void run() { - Future future = null; + Future future = null; try { future = executor.submit(task); threadPoolLevelGauge.set((long) executor.getActiveCount()); diff --git a/eureka-client/src/main/java/com/netflix/discovery/converters/EnumLookup.java b/eureka-client/src/main/java/com/netflix/discovery/converters/EnumLookup.java new file mode 100644 index 0000000000..8a69bc8eb5 --- /dev/null +++ b/eureka-client/src/main/java/com/netflix/discovery/converters/EnumLookup.java @@ -0,0 +1,104 @@ +package com.netflix.discovery.converters; + +import java.io.IOException; +import java.lang.reflect.Array; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Function; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; + +/** + * utility class for matching a Enum value to a region of a char[] without + * allocating any new objects on the heap. + */ +public class EnumLookup> { + private final int[] sortedHashes; + private final char[][] sortedNames; + private final Map stringLookup; + private final T[] sortedValues; + private final int minLength; + private final int maxLength; + + EnumLookup(Class enumType) { + this(enumType, t -> t.name().toCharArray()); + } + + @SuppressWarnings("unchecked") + EnumLookup(Class enumType, Function namer) { + this.sortedValues = (T[]) Array.newInstance(enumType, enumType.getEnumConstants().length); + System.arraycopy(enumType.getEnumConstants(), 0, sortedValues, 0, sortedValues.length); + Arrays.sort(sortedValues, + (o1, o2) -> Integer.compare(Arrays.hashCode(namer.apply(o1)), Arrays.hashCode(namer.apply(o2)))); + + this.sortedHashes = new int[sortedValues.length]; + this.sortedNames = new char[sortedValues.length][]; + int i = 0; + int minLength = Integer.MAX_VALUE; + int maxLength = Integer.MIN_VALUE; + stringLookup = new HashMap<>(); + for (T te : sortedValues) { + char[] name = namer.apply(te); + int hash = Arrays.hashCode(name); + sortedNames[i] = name; + sortedHashes[i++] = hash; + stringLookup.put(String.valueOf(name), te); + maxLength = Math.max(maxLength, name.length); + minLength = Math.min(minLength, name.length); + } + this.minLength = minLength; + this.maxLength = maxLength; + } + + public T find(JsonParser jp) throws IOException { + return find(jp, null); + } + + public T find(JsonParser jp, T defaultValue) throws IOException { + if (jp.getCurrentToken() == JsonToken.FIELD_NAME) { + return stringLookup.getOrDefault(jp.getCurrentName(), defaultValue); + } + return find(jp.getTextCharacters(), jp.getTextOffset(), jp.getTextLength(), defaultValue); + } + + public T find(char[] a, int offset, int length) { + return find(a, offset, length, null); + } + + public T find(char[] a, int offset, int length, T defaultValue) { + if (length < this.minLength || length > this.maxLength) return defaultValue; + + int hash = hashCode(a, offset, length); + int index = Arrays.binarySearch(sortedHashes, hash); + if (index >= 0) { + for (int i = index; i < sortedValues.length && sortedHashes[index] == hash; i++) { + if (equals(sortedNames[i], a, offset, length)) { + return sortedValues[i]; + } + } + } + return defaultValue; + } + + public static boolean equals(char[] a1, char[] a2, int a2Offset, int a2Length) { + if (a1.length != a2Length) + return false; + for (int i = 0; i < a2Length; i++) { + if (a1[i] != a2[i + a2Offset]) + return false; + } + return true; + } + + public static int hashCode(char[] a, int offset, int length) { + if (a == null) + return 0; + int result = 1; + for (int i = 0; i < length; i++) { + result = 31 * result + a[i + offset]; + } + return result; + } +} \ No newline at end of file diff --git a/eureka-client/src/main/java/com/netflix/discovery/converters/EurekaJacksonCodec.java b/eureka-client/src/main/java/com/netflix/discovery/converters/EurekaJacksonCodec.java index c3ae04dc4e..4a0cc9e997 100644 --- a/eureka-client/src/main/java/com/netflix/discovery/converters/EurekaJacksonCodec.java +++ b/eureka-client/src/main/java/com/netflix/discovery/converters/EurekaJacksonCodec.java @@ -8,26 +8,37 @@ import java.lang.reflect.Method; import java.util.Collections; import java.util.HashMap; -import java.util.Iterator; +import java.util.HashSet; import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.function.BiConsumer; +import java.util.function.Function; +import java.util.function.Supplier; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.JsonToken; import com.fasterxml.jackson.core.Version; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.JsonDeserializer; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.JsonSerializer; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.databind.RuntimeJsonMappingException; import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.jsontype.TypeSerializer; import com.fasterxml.jackson.databind.module.SimpleModule; -import com.fasterxml.jackson.databind.node.ArrayNode; import com.netflix.appinfo.AmazonInfo; import com.netflix.appinfo.DataCenterInfo; import com.netflix.appinfo.DataCenterInfo.Name; @@ -36,13 +47,13 @@ import com.netflix.appinfo.InstanceInfo.InstanceStatus; import com.netflix.appinfo.InstanceInfo.PortType; import com.netflix.appinfo.LeaseInfo; -import com.netflix.discovery.DiscoveryManager; import com.netflix.discovery.EurekaClientConfig; import com.netflix.discovery.shared.Application; import com.netflix.discovery.shared.Applications; -import com.netflix.discovery.util.StringCache; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import com.netflix.discovery.util.DeserializerStringCache; +import com.netflix.discovery.util.DeserializerStringCache.CacheScope; + +import vlsi.utils.CompactHashMap; /** * @author Tomasz Bak @@ -51,16 +62,14 @@ public class EurekaJacksonCodec { private static final Logger logger = LoggerFactory.getLogger(EurekaJacksonCodec.class); - private static final Version VERSION = new Version(1, 1, 0, null); - public static final String NODE_LEASE = "leaseInfo"; public static final String NODE_METADATA = "metadata"; public static final String NODE_DATACENTER = "dataCenterInfo"; public static final String NODE_APP = "application"; protected static final String ELEM_INSTANCE = "instance"; - protected static final String ELEM_OVERRIDDEN_STATUS = "overriddenstatus"; + protected static final String ELEM_OVERRIDDEN_STATUS = "overriddenStatus"; protected static final String ELEM_HOST = "hostName"; protected static final String ELEM_INSTANCE_ID = "instanceId"; protected static final String ELEM_APP = "app"; @@ -94,7 +103,7 @@ public class EurekaJacksonCodec { /** * XStream codec supports character replacement in field names to generate XML friendly * names. This feature is also configurable, and replacement strings can be provided by a user. - * To obey these rules, version and apppsHash key field names must be formatted according to the provided + * To obey these rules, version and appsHash key field names must be formatted according to the provided * configuration, which by default replaces '_' with '__' (double underscores). */ private final String versionDeltaKey; @@ -102,14 +111,22 @@ public class EurekaJacksonCodec { private final ObjectMapper mapper; - private final Map, ObjectReader> objectReaderByClass; + private final Map, Supplier> objectReaderByClass; private final Map, ObjectWriter> objectWriterByClass; + static EurekaClientConfig loadConfig() { + return com.netflix.discovery.DiscoveryManager.getInstance().getEurekaClientConfig(); + } + public EurekaJacksonCodec() { - this.versionDeltaKey = formatKey(VERSIONS_DELTA_TEMPLATE); - this.appHashCodeKey = formatKey(APPS_HASHCODE_TEMPTE); + this(formatKey(loadConfig(), VERSIONS_DELTA_TEMPLATE), formatKey(loadConfig(), APPS_HASHCODE_TEMPTE)); + + } + + public EurekaJacksonCodec(String versionDeltaKey, String appsHashCodeKey) { + this.versionDeltaKey = versionDeltaKey; + this.appHashCodeKey = appsHashCodeKey; this.mapper = new ObjectMapper(); - this.mapper.setSerializationInclusion(Include.NON_NULL); SimpleModule module = new SimpleModule("eureka1.x", VERSION); @@ -118,24 +135,22 @@ public EurekaJacksonCodec() { module.addSerializer(Application.class, new ApplicationSerializer()); module.addSerializer(Applications.class, new ApplicationsSerializer(this.versionDeltaKey, this.appHashCodeKey)); - module.addDeserializer(DataCenterInfo.class, new DataCenterInfoDeserializer()); module.addDeserializer(LeaseInfo.class, new LeaseInfoDeserializer()); module.addDeserializer(InstanceInfo.class, new InstanceInfoDeserializer(this.mapper)); module.addDeserializer(Application.class, new ApplicationDeserializer(this.mapper)); module.addDeserializer(Applications.class, new ApplicationsDeserializer(this.mapper, this.versionDeltaKey, this.appHashCodeKey)); - this.mapper.registerModule(module); - HashMap, ObjectReader> readers = new HashMap<>(); - readers.put(InstanceInfo.class, mapper.reader().withType(InstanceInfo.class).withRootName("instance")); - readers.put(Application.class, mapper.reader().withType(Application.class).withRootName("application")); - readers.put(Applications.class, mapper.reader().withType(Applications.class).withRootName("applications")); + Map, Supplier> readers = new HashMap<>(); + readers.put(InstanceInfo.class, ()->mapper.reader().forType(InstanceInfo.class).withRootName("instance")); + readers.put(Application.class, ()->mapper.reader().forType(Application.class).withRootName("application")); + readers.put(Applications.class, ()->mapper.reader().forType(Applications.class).withRootName("applications")); this.objectReaderByClass = readers; - HashMap, ObjectWriter> writers = new HashMap<>(); - writers.put(InstanceInfo.class, mapper.writer().withType(InstanceInfo.class).withRootName("instance")); - writers.put(Application.class, mapper.writer().withType(Application.class).withRootName("application")); - writers.put(Applications.class, mapper.writer().withType(Applications.class).withRootName("applications")); + Map, ObjectWriter> writers = new HashMap<>(); + writers.put(InstanceInfo.class, mapper.writer().forType(InstanceInfo.class).withRootName("instance")); + writers.put(Application.class, mapper.writer().forType(Application.class).withRootName("application")); + writers.put(Applications.class, mapper.writer().forType(Applications.class).withRootName("applications")); this.objectWriterByClass = writers; } @@ -151,8 +166,7 @@ protected String getAppHashCodeKey() { return appHashCodeKey; } - protected static String formatKey(String keyTemplate) { - EurekaClientConfig clientConfig = DiscoveryManager.getInstance().getEurekaClientConfig(); + protected static String formatKey(EurekaClientConfig clientConfig, String keyTemplate) { String replacement; if (clientConfig == null) { replacement = "__"; @@ -171,19 +185,27 @@ protected static String formatKey(String keyTemplate) { } public T readValue(Class type, InputStream entityStream) throws IOException { - ObjectReader reader = objectReaderByClass.get(type); - if (reader == null) { - return mapper.readValue(entityStream, type); + ObjectReader reader = DeserializerStringCache.init( + Optional.ofNullable(objectReaderByClass.get(type)).map(Supplier::get).orElseGet(()->mapper.readerFor(type)) + ); + try { + return reader.readValue(entityStream); + } + finally { + DeserializerStringCache.clear(reader, CacheScope.GLOBAL_SCOPE); } - return reader.readValue(entityStream); } public T readValue(Class type, String text) throws IOException { - ObjectReader reader = objectReaderByClass.get(type); - if (reader == null) { - return mapper.readValue(text, type); + ObjectReader reader = DeserializerStringCache.init( + Optional.ofNullable(objectReaderByClass.get(type)).map(Supplier::get).orElseGet(()->mapper.readerFor(type)) + ); + try { + return reader.readValue(text); + } + finally { + DeserializerStringCache.clear(reader, CacheScope.GLOBAL_SCOPE); } - return reader.readValue(text); } public void writeTo(T object, OutputStream entityStream) throws IOException { @@ -244,68 +266,52 @@ public void serialize(DataCenterInfo dataCenterInfo, JsonGenerator jgen, Seriali } } - public static class DataCenterInfoDeserializer extends JsonDeserializer { - - @Override - public DataCenterInfo deserialize(JsonParser jp, DeserializationContext context) throws IOException { - JsonNode node = jp.getCodec().readTree(jp); - final Name name = Name.valueOf(node.get(ELEM_NAME).asText()); - if (name != Name.Amazon) { - return new DataCenterInfo() { - @Override - public Name getName() { - return name; - } - }; + public static class LeaseInfoDeserializer extends JsonDeserializer { + enum LeaseInfoField { + DURATION("durationInSecs"), + EVICTION_TIMESTAMP("evictionTimestamp"), + LAST_RENEW_TIMESTAMP("lastRenewalTimestamp"), + REG_TIMESTAMP("registrationTimestamp"), + RENEW_INTERVAL("renewalIntervalInSecs"), + SERVICE_UP_TIMESTAMP("serviceUpTimestamp") + ; + private final char[] fieldName; + private LeaseInfoField(String fieldName) { + this.fieldName = fieldName.toCharArray(); } - - Map metaData = new HashMap(); - JsonNode metaNode = node.get(DATACENTER_METADATA); - Iterator metaNamesIt = metaNode.fieldNames(); - while (metaNamesIt.hasNext()) { - String key = metaNamesIt.next(); - String value = metaNode.get(key).asText(); - metaData.put(StringCache.intern(key), StringCache.intern(value)); + public char[] getFieldName() { + return fieldName; } - - AmazonInfo amazonInfo = new AmazonInfo(); - amazonInfo.setMetadata(metaData); - - return amazonInfo; } - } - - public static class LeaseInfoDeserializer extends JsonDeserializer { - - protected static final String ELEM_RENEW_INT = "renewalIntervalInSecs"; - protected static final String ELEM_DURATION = "durationInSecs"; - protected static final String ELEM_REG_TIMESTAMP = "registrationTimestamp"; - protected static final String ELEM_LAST_RENEW_TIMESTAMP = "lastRenewalTimestamp"; - protected static final String ELEM_EVICTION_TIMESTAMP = "evictionTimestamp"; - protected static final String ELEM_SERVICE_UP_TIMESTAMP = "serviceUpTimestamp"; + private static EnumLookup fieldLookup = new EnumLookup<>(LeaseInfoField.class, LeaseInfoField::getFieldName); @Override public LeaseInfo deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException { LeaseInfo.Builder builder = LeaseInfo.Builder.newBuilder(); - - JsonNode node = jp.getCodec().readTree(jp); - Iterator fieldNames = node.fieldNames(); - while (fieldNames.hasNext()) { - String nodeName = fieldNames.next(); - if (!node.get(nodeName).isNull()) { - long longValue = node.get(nodeName).asLong(); - if (ELEM_DURATION.equals(nodeName)) { - builder.setDurationInSecs((int) longValue); - } else if (ELEM_EVICTION_TIMESTAMP.equals(nodeName)) { - builder.setEvictionTimestamp(longValue); - } else if (ELEM_LAST_RENEW_TIMESTAMP.equals(nodeName)) { - builder.setRenewalTimestamp(longValue); - } else if (ELEM_REG_TIMESTAMP.equals(nodeName)) { - builder.setRegistrationTimestamp(longValue); - } else if (ELEM_RENEW_INT.equals(nodeName)) { - builder.setRenewalIntervalInSecs((int) longValue); - } else if (ELEM_SERVICE_UP_TIMESTAMP.equals(nodeName)) { - builder.setServiceUpTimestamp(longValue); + JsonToken jsonToken; + while ((jsonToken = jp.nextToken()) != JsonToken.END_OBJECT) { + LeaseInfoField field = fieldLookup.find(jp); + jsonToken = jp.nextToken(); + if (field != null && jsonToken != JsonToken.VALUE_NULL) { + switch(field) { + case DURATION: + builder.setDurationInSecs(jp.getValueAsInt()); + break; + case EVICTION_TIMESTAMP: + builder.setEvictionTimestamp(jp.getValueAsLong()); + break; + case LAST_RENEW_TIMESTAMP: + builder.setRenewalTimestamp(jp.getValueAsLong()); + break; + case REG_TIMESTAMP: + builder.setRegistrationTimestamp(jp.getValueAsLong()); + break; + case RENEW_INTERVAL: + builder.setRenewalIntervalInSecs(jp.getValueAsInt()); + break; + case SERVICE_UP_TIMESTAMP: + builder.setServiceUpTimestamp(jp.getValueAsLong()); + break; } } } @@ -330,8 +336,10 @@ public void serialize(InstanceInfo info, JsonGenerator jgen, SerializerProvider jgen.writeStringField(ELEM_APP, info.getAppName()); jgen.writeStringField(ELEM_IP, info.getIPAddr()); - if (!("unknown".equals(info.getSID()) || "na".equals(info.getSID()))) { - jgen.writeStringField(ELEM_SID, info.getSID()); + @SuppressWarnings("deprecation") + String sid = info.getSID(); + if (!("unknown".equals(sid) || "na".equals(sid))) { + jgen.writeStringField(ELEM_SID, sid); } jgen.writeStringField(ELEM_STATUS, info.getStatus().name()); @@ -373,7 +381,7 @@ public void serialize(InstanceInfo info, JsonGenerator jgen, SerializerProvider protected void autoMarshalEligible(Object o, JsonGenerator jgen) { try { - Class c = o.getClass(); + Class c = o.getClass(); Field[] fields = c.getDeclaredFields(); Annotation annotation; for (Field f : fields) { @@ -391,154 +399,281 @@ protected void autoMarshalEligible(Object o, JsonGenerator jgen) { } } } - + public static class InstanceInfoDeserializer extends JsonDeserializer { - protected ObjectMapper mapper; + private static char[] BUF_AT_CLASS = "@class".toCharArray(); + + enum InstanceInfoField { + HOSTNAME(ELEM_HOST), + INSTANCE_ID(ELEM_INSTANCE_ID), + APP(ELEM_APP), + IP(ELEM_IP), + SID(ELEM_SID), + ID_ATTR(ELEM_IDENTIFYING_ATTR),// nothing + STATUS(ELEM_STATUS), + OVERRIDDEN_STATUS(ELEM_OVERRIDDEN_STATUS), + PORT(ELEM_PORT), + SECURE_PORT(ELEM_SECURE_PORT), + COUNTRY_ID(ELEM_COUNTRY_ID), + DATACENTER(NODE_DATACENTER), + LEASE(NODE_LEASE), + HEALTHCHECKURL(ELEM_HEALTHCHECKURL), + SECHEALTHCHECKURL(ELEM_SECHEALTHCHECKURL), + APPGROUPNAME(ELEM_APPGROUPNAME), + HOMEPAGEURL(ELEM_HOMEPAGEURL), + STATUSPAGEURL(ELEM_STATUSPAGEURL), + VIPADDRESS(ELEM_VIPADDRESS), + SECVIPADDRESS(ELEM_SECVIPADDRESS), + ISCOORDINATINGDISCSERVER(ELEM_ISCOORDINATINGDISCSOERVER), + LASTUPDATEDTS(ELEM_LASTUPDATEDTS), + LASTDIRTYTS(ELEM_LASTDIRTYTS), + ACTIONTYPE(ELEM_ACTIONTYPE), + ASGNAME(ELEM_ASGNAME), + METADATA(NODE_METADATA) + ; + private final char[] elementName; + private InstanceInfoField(String elementName) { + this.elementName = elementName.toCharArray(); + } + public char[] getElementName() { + return elementName; + } + public static EnumLookup lookup = new EnumLookup<>(InstanceInfoField.class, InstanceInfoField::getElementName); + } + + enum PortField { + PORT("$"), ENABLED("@enabled"); + private final char[] fieldName; + private PortField(String name) { + this.fieldName = name.toCharArray(); + } + public char[] getFieldName() { return fieldName; } + public static EnumLookup lookup = new EnumLookup<>(PortField.class, PortField::getFieldName); + } + + private final ObjectMapper mapper; + private final ConcurrentMap> autoUnmarshalActions = new ConcurrentHashMap<>(); + private static EnumLookup statusLookup = new EnumLookup<>(InstanceStatus.class); + private static EnumLookup actionTypeLookup = new EnumLookup<>(ActionType.class); + static Set globalCachedMetadata = new HashSet<>(); + static { + globalCachedMetadata.add("route53Type"); + globalCachedMetadata.add("enableRoute53"); + globalCachedMetadata.add("netflix.stack"); + globalCachedMetadata.add("netflix.detail"); + globalCachedMetadata.add("NETFLIX_ENVIRONMENT"); + globalCachedMetadata.add("transportPort"); + } + protected InstanceInfoDeserializer(ObjectMapper mapper) { this.mapper = mapper; } + final static Function self = s->s; + @SuppressWarnings("deprecation") @Override public InstanceInfo deserialize(JsonParser jp, DeserializationContext context) throws IOException { - InstanceInfo.Builder builder = InstanceInfo.Builder.newBuilder(); - - JsonNode node = jp.getCodec().readTree(jp); - - /** - * These are set via single call to - * {@link com.netflix.appinfo.InstanceInfo.Builder#setHealthCheckUrlsForDeser(String, String, String)}. - */ - String healthChecUrl = null; - String healthCheckSecureUrl = null; - - Iterator fieldNames = node.fieldNames(); - while (fieldNames.hasNext()) { - String fieldName = fieldNames.next(); - JsonNode fieldNode = node.get(fieldName); - - if (!fieldNode.isNull()) { - if (ELEM_HOST.equals(fieldName)) { - builder.setHostName(fieldNode.asText()); - } else if (ELEM_INSTANCE_ID.equals(fieldName)) { - builder.setInstanceId(fieldNode.asText()); - } else if (ELEM_APP.equals(fieldName)) { - builder.setAppName(fieldNode.asText()); - } else if (ELEM_IP.equals(fieldName)) { - builder.setIPAddr(fieldNode.asText()); - } else if (ELEM_SID.equals(fieldName)) { - builder.setSID(fieldNode.asText()); - } else if (ELEM_IDENTIFYING_ATTR.equals(fieldName)) { - // nothing; - } else if (ELEM_STATUS.equals(fieldName)) { - builder.setStatus(InstanceStatus.toEnum(fieldNode.asText())); - } else if (ELEM_OVERRIDDEN_STATUS.equals(fieldName)) { - builder.setOverriddenStatus(InstanceStatus.toEnum(fieldNode.asText())); - } else if (ELEM_PORT.equals(fieldName)) { - int port = fieldNode.get("$").asInt(); - boolean enabled = fieldNode.get("@enabled").asBoolean(); - builder.setPort(port); - builder.enablePort(PortType.UNSECURE, enabled); - } else if (ELEM_SECURE_PORT.equals(fieldName)) { - int port = fieldNode.get("$").asInt(); - boolean enabled = fieldNode.get("@enabled").asBoolean(); - builder.setSecurePort(port); - builder.enablePort(PortType.SECURE, enabled); - } else if (ELEM_COUNTRY_ID.equals(fieldName)) { - builder.setCountryId(Integer.valueOf(fieldNode.asText()).intValue()); - } else if (NODE_DATACENTER.equals(fieldName)) { - builder.setDataCenterInfo(mapper.treeToValue(fieldNode, DataCenterInfo.class)); - } else if (NODE_LEASE.equals(fieldName)) { - builder.setLeaseInfo(mapper.treeToValue(fieldNode, LeaseInfo.class)); - } else if (NODE_METADATA.equals(fieldName)) { - Map meta = null; - Iterator metaNameIt = fieldNode.fieldNames(); - while (metaNameIt.hasNext()) { - String key = StringCache.intern(metaNameIt.next()); - if (key.equals("@class")) { // For backwards compatibility - if (meta == null && !metaNameIt.hasNext()) { // Optimize for empty maps - meta = Collections.emptyMap(); - } - } else { - if (meta == null) { - meta = new ConcurrentHashMap(); - } - String value = StringCache.intern(fieldNode.get(key).asText()); - meta.put(key, value); + if (Thread.currentThread().isInterrupted()) { + throw new JsonParseException(jp, "processing aborted"); + } + DeserializerStringCache intern = DeserializerStringCache.from(context); + InstanceInfo.Builder builder = InstanceInfo.Builder.newBuilder(self); + JsonToken jsonToken; + while ((jsonToken = jp.nextToken()) != JsonToken.END_OBJECT) { + InstanceInfoField instanceInfoField = InstanceInfoField.lookup.find(jp); + jsonToken = jp.nextToken(); + if (instanceInfoField != null && jsonToken != JsonToken.VALUE_NULL) { + switch(instanceInfoField) { + case HOSTNAME: + builder.setHostName(intern.apply(jp)); + break; + case INSTANCE_ID: + builder.setInstanceId(intern.apply(jp)); + break; + case APP: + builder.setAppNameForDeser( + intern.apply(jp, CacheScope.APPLICATION_SCOPE, + ()->{ + try { + return jp.getText().toUpperCase(); + } catch (IOException e) { + throw new RuntimeJsonMappingException(e.getMessage()); + } + })); + break; + case IP: + builder.setIPAddr(intern.apply(jp)); + break; + case SID: + builder.setSID(intern.apply(jp, CacheScope.GLOBAL_SCOPE)); + break; + case ID_ATTR: + // nothing + break; + case STATUS: + builder.setStatus(statusLookup.find(jp, InstanceStatus.UNKNOWN)); + break; + case OVERRIDDEN_STATUS: + builder.setOverriddenStatus(statusLookup.find(jp, InstanceStatus.UNKNOWN)); + break; + case PORT: + while ((jsonToken = jp.nextToken()) != JsonToken.END_OBJECT) { + PortField field = PortField.lookup.find(jp); + switch(field) { + case PORT: + if (jsonToken == JsonToken.FIELD_NAME) jp.nextToken(); + builder.setPort(jp.getValueAsInt()); + break; + case ENABLED: + if (jsonToken == JsonToken.FIELD_NAME) jp.nextToken(); + builder.enablePort(PortType.UNSECURE, jp.getValueAsBoolean()); + break; + default: } } - if (meta == null) { - meta = Collections.emptyMap(); + break; + case SECURE_PORT: + while ((jsonToken = jp.nextToken()) != JsonToken.END_OBJECT) { + PortField field = PortField.lookup.find(jp); + switch(field) { + case PORT: + if (jsonToken == JsonToken.FIELD_NAME) jp.nextToken(); + builder.setSecurePort(jp.getValueAsInt()); + break; + case ENABLED: + if (jsonToken == JsonToken.FIELD_NAME) jp.nextToken(); + builder.enablePort(PortType.SECURE, jp.getValueAsBoolean()); + break; + default: + } } - builder.setMetadata(meta); - } else if (ELEM_HEALTHCHECKURL.equals(fieldName)) { - healthChecUrl = fieldNode.asText(); - } else if (ELEM_SECHEALTHCHECKURL.equals(fieldName)) { - healthCheckSecureUrl = fieldNode.asText(); - } else if (ELEM_APPGROUPNAME.equals(fieldName)) { - builder.setAppGroupName(fieldNode.asText()); - } else if (ELEM_HOMEPAGEURL.equals(fieldName)) { - builder.setHomePageUrlForDeser(fieldNode.asText()); - } else if (ELEM_STATUSPAGEURL.equals(fieldName)) { - builder.setStatusPageUrlForDeser(fieldNode.asText()); - } else if (ELEM_VIPADDRESS.equals(fieldName)) { - builder.setVIPAddressDeser(fieldNode.asText()); - } else if (ELEM_SECVIPADDRESS.equals(fieldName)) { - builder.setSecureVIPAddressDeser(fieldNode.asText()); - } else if (ELEM_ISCOORDINATINGDISCSOERVER.equals(fieldName)) { - builder.setIsCoordinatingDiscoveryServer(fieldNode.asBoolean()); - } else if (ELEM_LASTUPDATEDTS.equals(fieldName)) { - builder.setLastUpdatedTimestamp(fieldNode.asLong()); - } else if (ELEM_LASTDIRTYTS.equals(fieldName)) { - builder.setLastDirtyTimestamp(fieldNode.asLong()); - } else if (ELEM_ACTIONTYPE.equals(fieldName)) { - builder.setActionType(ActionType.valueOf(fieldNode.asText())); - } else if (ELEM_ASGNAME.equals(fieldName)) { - builder.setASGName(fieldNode.asText()); - } else { - autoUnmarshalEligible(fieldName, fieldNode.asText(), builder.getRawInstance()); + break; + case COUNTRY_ID: + builder.setCountryId(jp.getValueAsInt()); + break; + case DATACENTER: + builder.setDataCenterInfo(DeserializerStringCache.init(mapper.readerFor(DataCenterInfo.class), context).readValue(jp)); + break; + case LEASE: + builder.setLeaseInfo(mapper.readerFor(LeaseInfo.class).readValue(jp)); + break; + case HEALTHCHECKURL: + builder.setHealthCheckUrlsForDeser(intern.apply(jp.getText()), null); + break; + case SECHEALTHCHECKURL: + builder.setHealthCheckUrlsForDeser(null, intern.apply(jp.getText())); + break; + case APPGROUPNAME: + builder.setAppGroupNameForDeser(intern.apply(jp, CacheScope.GLOBAL_SCOPE, + ()->{ + try { + return jp.getText().toUpperCase(); + } catch (IOException e) { + throw new RuntimeJsonMappingException(e.getMessage()); + } + })); + break; + case HOMEPAGEURL: + builder.setHomePageUrlForDeser(intern.apply(jp.getText())); + break; + case STATUSPAGEURL: + builder.setStatusPageUrlForDeser(intern.apply(jp.getText())); + break; + case VIPADDRESS: + builder.setVIPAddressDeser(intern.apply(jp)); + break; + case SECVIPADDRESS: + builder.setSecureVIPAddressDeser(intern.apply(jp)); + break; + case ISCOORDINATINGDISCSERVER: + builder.setIsCoordinatingDiscoveryServer(jp.getValueAsBoolean()); + break; + case LASTUPDATEDTS: + builder.setLastUpdatedTimestamp(jp.getValueAsLong()); + break; + case LASTDIRTYTS: + builder.setLastDirtyTimestamp(jp.getValueAsLong()); + break; + case ACTIONTYPE: + builder.setActionType(actionTypeLookup.find(jp.getTextCharacters(), jp.getTextOffset(), jp.getTextLength())); + break; + case ASGNAME: + builder.setASGName(intern.apply(jp)); + break; + case METADATA: + Map metadataMap = null; + while ((jsonToken = jp.nextToken()) != JsonToken.END_OBJECT) { + char[] parserChars = jp.getTextCharacters(); + if (parserChars[0] == '@' && EnumLookup.equals(BUF_AT_CLASS, parserChars, jp.getTextOffset(), jp.getTextLength())) { + // skip this + jsonToken = jp.nextToken(); + } + else { // For backwards compatibility + String key = intern.apply(jp, CacheScope.GLOBAL_SCOPE); + jsonToken = jp.nextToken(); + String value = intern.apply(jp, CacheScope.APPLICATION_SCOPE ); + metadataMap = Optional.ofNullable(metadataMap).orElseGet(CompactHashMap::new); + metadataMap.put(key, value); + } + }; + builder.setMetadata(metadataMap == null ? Collections.emptyMap() : Collections.synchronizedMap(metadataMap)); + break; + default: + autoUnmarshalEligible(jp.getCurrentName(), jp.getValueAsString(), builder.getRawInstance()); } } + else { + autoUnmarshalEligible(jp.getCurrentName(), jp.getValueAsString(), builder.getRawInstance()); + } } - builder.setHealthCheckUrlsForDeser(healthChecUrl, healthCheckSecureUrl); - return builder.build(); } - protected void autoUnmarshalEligible(String fieldName, String value, Object o) { - try { - Class c = o.getClass(); - Field f = null; + void autoUnmarshalEligible(String fieldName, String value, Object o) { + if (value == null || o == null) return; // early out + Class c = o.getClass(); + String cacheKey = c.getName() + ":" + fieldName; + BiConsumer action = autoUnmarshalActions.computeIfAbsent(cacheKey, k-> { try { - f = c.getDeclaredField(fieldName); - } catch (NoSuchFieldException e) { - // TODO XStream version increments metrics counter here - } - if (f == null) { - return; - } - Annotation annotation = f.getAnnotation(Auto.class); - if (annotation == null) { - return; - } - f.setAccessible(true); + Field f = null; + try { + f = c.getDeclaredField(fieldName); + } catch (NoSuchFieldException e) { + // TODO XStream version increments metrics counter here + } + if (f == null) { + return (t,v)->{}; + } + Annotation annotation = f.getAnnotation(Auto.class); + if (annotation == null) { + return (t,v)->{}; + } + f.setAccessible(true); - Class returnClass = f.getType(); - if (value != null) { + final Field setterField = f; + Class returnClass = setterField.getType(); if (!String.class.equals(returnClass)) { Method method = returnClass.getDeclaredMethod("valueOf", java.lang.String.class); - Object valueObject = method.invoke(returnClass, value); - f.set(o, valueObject); + return (t, v) -> tryCatchLog(()->{ setterField.set(t, method.invoke(returnClass, v)); return null; }); } else { - f.set(o, value); - - } + return (t, v) -> tryCatchLog(()->{ setterField.set(t, v); return null; }); + } + } catch (Exception ex) { + logger.error("Error in unmarshalling the object:", ex); + return null; } - } catch (Throwable th) { - logger.error("Error in unmarshalling the object:", th); - } + }); + action.accept(o, value); } + } + private static void tryCatchLog(Callable callable) { + try { + callable.call(); + } catch (Exception ex) { + logger.error("Error in unmarshalling the object:", ex); + } } public static class ApplicationSerializer extends JsonSerializer { @@ -552,32 +687,60 @@ public void serialize(Application value, JsonGenerator jgen, SerializerProvider } public static class ApplicationDeserializer extends JsonDeserializer { + enum ApplicationField { + NAME(ELEM_NAME), INSTANCE(ELEM_INSTANCE); + private final char[] fieldName; + private ApplicationField(String name) { + this.fieldName = name.toCharArray(); + } + public char[] getFieldName() { return fieldName; } + public static EnumLookup lookup = new EnumLookup<>(ApplicationField.class, ApplicationField::getFieldName); + + } - protected ObjectMapper mapper; + private final ObjectMapper mapper; public ApplicationDeserializer(ObjectMapper mapper) { this.mapper = mapper; } - + @Override public Application deserialize(JsonParser jp, DeserializationContext context) throws IOException { - JsonNode node = jp.getCodec().readTree(jp); - - Application application = new Application(node.get(ELEM_NAME).asText()); - - JsonNode instanceNode = node.get(ELEM_INSTANCE); - if (instanceNode != null) { - if (instanceNode instanceof ArrayNode) { - ArrayNode instancesNode = (ArrayNode) instanceNode; - if (instancesNode != null) { - for (JsonNode nextNode : instancesNode) { - application.addInstance(mapper.treeToValue(nextNode, InstanceInfo.class)); - } + if (Thread.currentThread().isInterrupted()) { + throw new JsonParseException(jp, "processing aborted"); + } + Application application = new Application(); + JsonToken jsonToken; + try { + while((jsonToken = jp.nextToken()) != JsonToken.END_OBJECT){ + if(JsonToken.FIELD_NAME == jsonToken){ + ApplicationField field = ApplicationField.lookup.find(jp); + jsonToken = jp.nextToken(); + if (field != null) { + switch(field) { + case NAME: + application.setName(jp.getText()); + break; + case INSTANCE: + ObjectReader instanceInfoReader = DeserializerStringCache.init(mapper.readerFor(InstanceInfo.class), context); + if (jsonToken == JsonToken.START_ARRAY) { + // messages is array, loop until token equal to "]" + while (jp.nextToken() != JsonToken.END_ARRAY) { + application.addInstance(instanceInfoReader.readValue(jp)); + } + } + else if (jsonToken == JsonToken.START_OBJECT) { + application.addInstance(instanceInfoReader.readValue(jp)); + } + break; + } + } } - } else { - application.addInstance(mapper.treeToValue(instanceNode, InstanceInfo.class)); } } + finally { +// DeserializerStringCache.clear(context, CacheScope.APPLICATION_SCOPE); + } return application; } } @@ -613,28 +776,37 @@ public ApplicationsDeserializer(ObjectMapper mapper, String versionDeltaKey, Str @Override public Applications deserialize(JsonParser jp, DeserializationContext context) throws IOException { - Applications apps = new Applications(); - - JsonNode node = jp.getCodec().readTree(jp); - - if (node.get(versionDeltaKey) != null) { - apps.setVersion(node.get(versionDeltaKey).asLong()); + if (Thread.currentThread().isInterrupted()) { + throw new JsonParseException(jp, "processing aborted"); } - if (node.get(appHashCodeKey) != null) { - apps.setAppsHashCode(node.get(appHashCodeKey).asText()); - } - JsonNode appNode = node.get(NODE_APP); - if (appNode != null) { - if (appNode instanceof ArrayNode) { - ArrayNode appsNode = (ArrayNode) appNode; - for (JsonNode item : appsNode) { - apps.addApplication(mapper.treeToValue(item, Application.class)); + Applications apps = new Applications(); + JsonToken jsonToken; + while((jsonToken = jp.nextToken()) != JsonToken.END_OBJECT){ + + if(JsonToken.FIELD_NAME == jsonToken){ + String fieldName = jp.getCurrentName(); + jsonToken = jp.nextToken(); + + if(versionDeltaKey.equals(fieldName)){ + apps.setVersion(jp.getValueAsLong()); + } else if (appHashCodeKey.equals(fieldName)){ + apps.setAppsHashCode(jp.getValueAsString()); + } + else if (NODE_APP.equals(fieldName)) { + ObjectReader applicationReader = DeserializerStringCache.init(mapper.readerFor(Application.class), context); + if (jsonToken == JsonToken.START_ARRAY) { + while (jp.nextToken() != JsonToken.END_ARRAY) { + apps.addApplication(applicationReader.readValue(jp)); + } + } + else if (jsonToken == JsonToken.START_OBJECT) { + apps.addApplication(applicationReader.readValue(jp)); + } } - } else { - apps.addApplication(mapper.treeToValue(appNode, Application.class)); } } return apps; } } + } diff --git a/eureka-client/src/main/java/com/netflix/discovery/converters/jackson/builder/StringInterningAmazonInfoBuilder.java b/eureka-client/src/main/java/com/netflix/discovery/converters/jackson/builder/StringInterningAmazonInfoBuilder.java index 0ea0c0015d..bf19aae7d1 100644 --- a/eureka-client/src/main/java/com/netflix/discovery/converters/jackson/builder/StringInterningAmazonInfoBuilder.java +++ b/eureka-client/src/main/java/com/netflix/discovery/converters/jackson/builder/StringInterningAmazonInfoBuilder.java @@ -16,16 +16,24 @@ package com.netflix.discovery.converters.jackson.builder; +import java.io.IOException; import java.util.HashMap; -import java.util.HashSet; import java.util.Map; -import java.util.Set; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; import com.netflix.appinfo.AmazonInfo; import com.netflix.appinfo.AmazonInfo.MetaDataKey; import com.netflix.appinfo.DataCenterInfo.Name; +import com.netflix.discovery.converters.EnumLookup; +import com.netflix.discovery.util.DeserializerStringCache; +import com.netflix.discovery.util.DeserializerStringCache.CacheScope; import com.netflix.discovery.util.StringCache; +import vlsi.utils.CompactHashMap; + /** * Amazon instance info builder that is doing key names interning, together with * value interning for selected keys (see {@link StringInterningAmazonInfoBuilder#VALUE_INTERN_KEYS}). @@ -35,17 +43,20 @@ * * @author Tomasz Bak */ -public class StringInterningAmazonInfoBuilder { +public class StringInterningAmazonInfoBuilder extends JsonDeserializer{ - private static final Set VALUE_INTERN_KEYS; + private static final Map VALUE_INTERN_KEYS; + private static final char[] BUF_METADATA = "metadata".toCharArray(); static { - HashSet keys = new HashSet<>(); - keys.add(MetaDataKey.accountId.getName()); - keys.add(MetaDataKey.amiId.getName()); - keys.add(MetaDataKey.availabilityZone.getName()); - keys.add(MetaDataKey.instanceType.getName()); - keys.add(MetaDataKey.vpcId.getName()); + HashMap keys = new HashMap<>(); + keys.put(MetaDataKey.accountId.getName(), CacheScope.GLOBAL_SCOPE); + keys.put(MetaDataKey.amiId.getName(), CacheScope.GLOBAL_SCOPE); + keys.put(MetaDataKey.availabilityZone.getName(), CacheScope.GLOBAL_SCOPE); + keys.put(MetaDataKey.instanceType.getName(), CacheScope.GLOBAL_SCOPE); + keys.put(MetaDataKey.vpcId.getName(), CacheScope.GLOBAL_SCOPE); + keys.put(MetaDataKey.publicIpv4.getName(), CacheScope.APPLICATION_SCOPE); + keys.put(MetaDataKey.localHostname.getName(), CacheScope.APPLICATION_SCOPE); VALUE_INTERN_KEYS = keys; } @@ -56,18 +67,15 @@ public StringInterningAmazonInfoBuilder withName(String name) { } public StringInterningAmazonInfoBuilder withMetadata(HashMap metadata) { + this.metadata = metadata; if (metadata.isEmpty()) { - this.metadata = metadata; return this; } - this.metadata = new HashMap<>(); for (Map.Entry entry : metadata.entrySet()) { String key = entry.getKey().intern(); - String value = entry.getValue(); - if (VALUE_INTERN_KEYS.contains(key)) { - value = StringCache.intern(value); + if (VALUE_INTERN_KEYS.containsKey(key)) { + entry.setValue(StringCache.intern(entry.getValue())); } - this.metadata.put(key, value); } return this; } @@ -75,4 +83,32 @@ public StringInterningAmazonInfoBuilder withMetadata(HashMap met public AmazonInfo build() { return new AmazonInfo(Name.Amazon.name(), metadata); } + + @Override + public AmazonInfo deserialize(JsonParser jp, DeserializationContext context) + throws IOException { + Map metadata = new CompactHashMap<>(); + DeserializerStringCache intern = DeserializerStringCache.from(context); + + JsonToken jsonToken; + while((jsonToken = jp.nextToken()) != JsonToken.END_OBJECT){ + jsonToken = jp.nextToken(); + + if (EnumLookup.equals(BUF_METADATA, jp.getTextCharacters(), jp.getTextOffset(), jp.getTextLength())) { + jsonToken = jp.nextToken(); + while((jsonToken = jp.nextToken()) != JsonToken.END_OBJECT) { + String metadataKey = intern.apply(jp, CacheScope.GLOBAL_SCOPE); + jp.nextToken(); + CacheScope scope = VALUE_INTERN_KEYS.get(metadataKey); + String metadataValue = (scope != null) ? intern.apply(jp, scope) : intern.apply(jp, CacheScope.APPLICATION_SCOPE); + metadata.put(metadataKey, metadataValue); + } + } + else { + jsonToken = jp.nextToken(); + } + } + return new AmazonInfo(Name.Amazon.name(), metadata); + } + } diff --git a/eureka-client/src/main/java/com/netflix/discovery/shared/Application.java b/eureka-client/src/main/java/com/netflix/discovery/shared/Application.java index 9b4d161b01..e069298ca5 100755 --- a/eureka-client/src/main/java/com/netflix/discovery/shared/Application.java +++ b/eureka-client/src/main/java/com/netflix/discovery/shared/Application.java @@ -23,6 +23,8 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.Random; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicReference; @@ -52,6 +54,8 @@ @XStreamAlias("application") @JsonRootName("application") public class Application { + + private static Random shuffleRandom = new Random(); @Override public String toString() { @@ -68,19 +72,19 @@ public String toString() { @XStreamImplicit private final Set instances; - private AtomicReference> shuffledInstances = new AtomicReference>(); + private final AtomicReference> shuffledInstances; - private Map instancesMap; + private final Map instancesMap; public Application() { instances = new LinkedHashSet(); instancesMap = new ConcurrentHashMap(); + shuffledInstances = new AtomicReference>(); } public Application(String name) { + this(); this.name = StringCache.intern(name); - instancesMap = new ConcurrentHashMap(); - instances = new LinkedHashSet(); } @JsonCreator @@ -131,11 +135,7 @@ public void removeInstance(InstanceInfo i) { */ @JsonProperty("instance") public List getInstances() { - if (this.shuffledInstances.get() == null) { - return this.getInstancesAsIsFromEureka(); - } else { - return this.shuffledInstances.get(); - } + return Optional.ofNullable(shuffledInstances.get()).orElseGet(this::getInstancesAsIsFromEureka); } /** @@ -216,14 +216,15 @@ private void _shuffleAndStoreInstances(boolean filterUpInstances, boolean indexB synchronized (instances) { instanceInfoList = new ArrayList(instances); } - if (indexByRemoteRegions || filterUpInstances) { + boolean remoteIndexingActive = indexByRemoteRegions && null != instanceRegionChecker && null != clientConfig + && null != remoteRegionsRegistry; + if (remoteIndexingActive || filterUpInstances) { Iterator it = instanceInfoList.iterator(); while (it.hasNext()) { InstanceInfo instanceInfo = it.next(); - if (filterUpInstances && !InstanceStatus.UP.equals(instanceInfo.getStatus())) { + if (filterUpInstances && InstanceStatus.UP != instanceInfo.getStatus()) { it.remove(); - } else if (indexByRemoteRegions && null != instanceRegionChecker && null != clientConfig - && null != remoteRegionsRegistry) { + } else if (remoteIndexingActive) { String instanceRegion = instanceRegionChecker.getInstanceRegion(instanceInfo); if (!instanceRegionChecker.isLocalRegion(instanceRegion)) { Applications appsForRemoteRegion = remoteRegionsRegistry.get(instanceRegion); @@ -247,7 +248,7 @@ private void _shuffleAndStoreInstances(boolean filterUpInstances, boolean indexB } } - Collections.shuffle(instanceInfoList); + Collections.shuffle(instanceInfoList, shuffleRandom); this.shuffledInstances.set(instanceInfoList); } diff --git a/eureka-client/src/main/java/com/netflix/discovery/shared/Applications.java b/eureka-client/src/main/java/com/netflix/discovery/shared/Applications.java index fe5599ead0..17d7af0192 100755 --- a/eureka-client/src/main/java/com/netflix/discovery/shared/Applications.java +++ b/eureka-client/src/main/java/com/netflix/discovery/shared/Applications.java @@ -20,40 +20,38 @@ import java.util.AbstractQueue; import java.util.ArrayList; import java.util.Collections; -import java.util.HashSet; -import java.util.Iterator; +import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Set; +import java.util.Optional; +import java.util.Random; import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonRootName; import com.netflix.appinfo.InstanceInfo; -import com.netflix.appinfo.InstanceInfo.ActionType; import com.netflix.appinfo.InstanceInfo.InstanceStatus; import com.netflix.discovery.EurekaClientConfig; import com.netflix.discovery.InstanceRegionChecker; import com.netflix.discovery.provider.Serializer; import com.thoughtworks.xstream.annotations.XStreamAlias; import com.thoughtworks.xstream.annotations.XStreamImplicit; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * The class that wraps all the registry information returned by eureka server. * *

* Note that the registry information is fetched from eureka server as specified - * in {@link EurekaClientConfig#getRegistryFetchIntervalSeconds()}. Once the + * in {@link EurekaClientConfig#getRegistryFetchIntervalSeconds()}. Once the * information is fetched it is shuffled and also filtered for instances with * {@link InstanceStatus#UP} status as specified by the configuration * {@link EurekaClientConfig#shouldFilterOnlyUpInstances()}. @@ -66,58 +64,55 @@ @XStreamAlias("applications") @JsonRootName("applications") public class Applications { - private static final String APP_INSTANCEID_DELIMITER = "$$"; - private static final Logger logger = LoggerFactory.getLogger(Applications.class); - private static final String STATUS_DELIMITER = "_"; - - private Long versionDelta = Long.valueOf(-1); + private static class VipIndexSupport { + final AbstractQueue instances = new ConcurrentLinkedQueue<>(); + final AtomicLong roundRobinIndex = new AtomicLong(0); + final AtomicReference> vipList = new AtomicReference>(Collections.emptyList()); - @XStreamImplicit - private AbstractQueue applications; + public AtomicLong getRoundRobinIndex() { + return roundRobinIndex; + } - private Map appNameApplicationMap = new ConcurrentHashMap(); - private Map> virtualHostNameAppMap = new ConcurrentHashMap>(); - private Map> secureVirtualHostNameAppMap = new ConcurrentHashMap>(); - private Map virtualHostNameIndexMap = new ConcurrentHashMap(); - private Map secureVirtualHostNameIndexMap = new ConcurrentHashMap(); + public AtomicReference> getVipList() { + return vipList; + } + } - private Map>> shuffleVirtualHostNameMap = new ConcurrentHashMap>>(); - private Map>> shuffledSecureVirtualHostNameMap = new ConcurrentHashMap>>(); + private static final String STATUS_DELIMITER = "_"; private String appsHashCode; + private Long versionDelta; + @XStreamImplicit + private final AbstractQueue applications; + private final Map appNameApplicationMap; + private final Map virtualHostNameAppMap; + private final Map secureVirtualHostNameAppMap; /** * Create a new, empty Eureka application list. */ public Applications() { - this.applications = new ConcurrentLinkedQueue(); + this(null, -1L, Collections.emptyList()); } /** - * Note that appsHashCode and versionDelta key names are formatted in a custom/configurable way. + * Note that appsHashCode and versionDelta key names are formatted in a + * custom/configurable way. */ @JsonCreator - public Applications( - @JsonProperty("appsHashCode") String appsHashCode, + public Applications(@JsonProperty("appsHashCode") String appsHashCode, @JsonProperty("versionDelta") Long versionDelta, @JsonProperty("application") List registeredApplications) { this.applications = new ConcurrentLinkedQueue(); - for (Application app : registeredApplications) { - this.addApplication(app); - } + this.appNameApplicationMap = new ConcurrentHashMap(); + this.virtualHostNameAppMap = new ConcurrentHashMap(); + this.secureVirtualHostNameAppMap = new ConcurrentHashMap(); this.appsHashCode = appsHashCode; this.versionDelta = versionDelta; - } - /** - * Create a new Eureka application list, based on the provided applications. The provided container is - * not modified. - * - * @param apps the initial list of apps to store in this applications list - */ - public Applications(List apps) { - this.applications = new ConcurrentLinkedQueue(); - this.applications.addAll(apps); + for (Application app : registeredApplications) { + this.addApplication(app); + } } /** @@ -128,11 +123,10 @@ public Applications(List apps) { */ public void addApplication(Application app) { appNameApplicationMap.put(app.getName().toUpperCase(Locale.ROOT), app); - addInstancesToVIPMaps(app); + addInstancesToVIPMaps(app, this.virtualHostNameAppMap, this.secureVirtualHostNameAppMap); applications.add(app); } - /** * Gets the list of all registered applications from eureka. * @@ -140,9 +134,7 @@ public void addApplication(Application app) { */ @JsonProperty("application") public List getRegisteredApplications() { - List list = new ArrayList(); - list.addAll(this.applications); - return list; + return new ArrayList(this.applications); } /** @@ -167,13 +159,10 @@ public Application getRegisteredApplications(String appName) { * @return list of instances. */ public List getInstancesByVirtualHostName(String virtualHostName) { - AtomicReference> ref = this.shuffleVirtualHostNameMap - .get(virtualHostName.toUpperCase(Locale.ROOT)); - if (ref == null || ref.get() == null) { - return new ArrayList(); - } else { - return ref.get(); - } + return Optional.ofNullable(this.virtualHostNameAppMap.get(virtualHostName.toUpperCase(Locale.ROOT))) + .map(VipIndexSupport::getVipList) + .map(AtomicReference::get) + .orElseGet(Collections::emptyList); } /** @@ -186,25 +175,18 @@ public List getInstancesByVirtualHostName(String virtualHostName) * @return list of instances. */ public List getInstancesBySecureVirtualHostName(String secureVirtualHostName) { - AtomicReference> ref = this.shuffledSecureVirtualHostNameMap - .get(secureVirtualHostName.toUpperCase(Locale.ROOT)); - if (ref == null || ref.get() == null) { - return new ArrayList(); - } else { - return ref.get(); - } + return Optional.ofNullable(this.secureVirtualHostNameAppMap.get(secureVirtualHostName.toUpperCase(Locale.ROOT))) + .map(VipIndexSupport::getVipList) + .map(AtomicReference::get) + .orElseGet(Collections::emptyList); } /** - * @return a weakly consistent size of the number of instances in all the applications + * @return a weakly consistent size of the number of instances in all the + * applications */ public int size() { - int result = 0; - for (Application application : applications) { - result += application.size(); - } - - return result; + return applications.stream().mapToInt(Application::size).sum(); } @Deprecated @@ -221,7 +203,8 @@ public Long getVersion() { /** * Used by the eureka server. Not for external use. * - * @param hashCode the hash code to assign for this app collection + * @param hashCode + * the hash code to assign for this app collection */ public void setAppsHashCode(String hashCode) { this.appsHashCode = hashCode; @@ -229,7 +212,9 @@ public void setAppsHashCode(String hashCode) { /** * Used by the eureka server. Not for external use. - * @return the string indicating the hashcode based on the applications stored. + * + * @return the string indicating the hashcode based on the applications + * stored. * */ @JsonIgnore // Handled directly due to legacy name formatting @@ -252,201 +237,91 @@ public String getReconcileHashCode() { } /** - * Populates the provided instance count map. The instance count map is used as part of the general - * app list synchronization mechanism. - * @param instanceCountMap the map to populate + * Populates the provided instance count map. The instance count map is used + * as part of the general app list synchronization mechanism. + * + * @param instanceCountMap + * the map to populate */ - public void populateInstanceCountMap(TreeMap instanceCountMap) { + public void populateInstanceCountMap(Map instanceCountMap) { for (Application app : this.getRegisteredApplications()) { for (InstanceInfo info : app.getInstancesAsIsFromEureka()) { - AtomicInteger instanceCount = instanceCountMap.get(info.getStatus().name()); - if (instanceCount == null) { - instanceCount = new AtomicInteger(0); - instanceCountMap.put(info.getStatus().name(), instanceCount); - } + AtomicInteger instanceCount = instanceCountMap.computeIfAbsent(info.getStatus().name(), + k -> new AtomicInteger(0)); instanceCount.incrementAndGet(); } } } /** - * Gets the reconciliation hashcode. The hashcode is used to determine whether the applications list - * has changed since the last time it was acquired. - * @param instanceCountMap the instance count map to use for generating the hash + * Gets the reconciliation hashcode. The hashcode is used to determine + * whether the applications list has changed since the last time it was + * acquired. + * + * @param instanceCountMap + * the instance count map to use for generating the hash * @return the hash code for this instance */ - public static String getReconcileHashCode(TreeMap instanceCountMap) { - String reconcileHashCode = ""; + public static String getReconcileHashCode(Map instanceCountMap) { + StringBuilder reconcileHashCode = new StringBuilder(75); for (Map.Entry mapEntry : instanceCountMap.entrySet()) { - reconcileHashCode = reconcileHashCode + mapEntry.getKey() - + STATUS_DELIMITER + mapEntry.getValue().get() - + STATUS_DELIMITER; - } - return reconcileHashCode; - } - - /** - * Gets the exact difference between this applications instance and another - * one. - * - * @param apps - * the applications for which to compare this one. - * @return a map containing the differences between the two. - */ - public Map> getReconcileMapDiff(Applications apps) { - Map> diffMap = new TreeMap>(); - Set allInstanceAppInstanceIds = new HashSet(); - for (Application otherApp : apps.getRegisteredApplications()) { - Application thisApp = this.getRegisteredApplications(otherApp.getName()); - if (thisApp == null) { - logger.warn("Application not found in local cache : {}", otherApp.getName()); - continue; - } - for (InstanceInfo instanceInfo : thisApp.getInstancesAsIsFromEureka()) { - allInstanceAppInstanceIds.add(new Pair(thisApp.getName(), - instanceInfo.getId())); - } - for (InstanceInfo otherInstanceInfo : otherApp.getInstancesAsIsFromEureka()) { - InstanceInfo thisInstanceInfo = thisApp.getByInstanceId(otherInstanceInfo.getId()); - if (thisInstanceInfo == null) { - List diffList = diffMap.get(ActionType.DELETED.name()); - if (diffList == null) { - diffList = new ArrayList(); - diffMap.put(ActionType.DELETED.name(), diffList); - } - diffList.add(otherInstanceInfo.getId()); - } else if (!thisInstanceInfo.getStatus().name() - .equalsIgnoreCase(otherInstanceInfo.getStatus().name())) { - List diffList = diffMap.get(ActionType.MODIFIED.name()); - if (diffList == null) { - diffList = new ArrayList(); - diffMap.put(ActionType.MODIFIED.name(), diffList); - } - diffList.add(thisInstanceInfo.getId() - + APP_INSTANCEID_DELIMITER - + thisInstanceInfo.getStatus().name() - + APP_INSTANCEID_DELIMITER - + otherInstanceInfo.getStatus().name()); - } - allInstanceAppInstanceIds.remove(new Pair(otherApp.getName(), otherInstanceInfo.getId())); - } - } - for (Pair pair : allInstanceAppInstanceIds) { - Application app = new Application(pair.getItem1()); - InstanceInfo thisInstanceInfo = app.getByInstanceId(pair.getItem2()); - if (thisInstanceInfo != null) { - List diffList = diffMap.get(ActionType.ADDED.name()); - if (diffList == null) { - diffList = new ArrayList(); - diffMap.put(ActionType.ADDED.name(), diffList); - } - diffList.add(thisInstanceInfo.getId()); - } - } - return diffMap; - - } - - private static final class Pair { - private final String item1; - private final String item2; - - public Pair(String item1, String item2) { - super(); - this.item1 = item1; - this.item2 = item2; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result - + ((item1 == null) ? 0 : item1.hashCode()); - result = prime * result - + ((item2 == null) ? 0 : item2.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - Pair other = (Pair) obj; - if (item1 == null) { - if (other.item1 != null) { - return false; - } - } else if (!item1.equals(other.item1)) { - return false; - } - if (item2 == null) { - if (other.item2 != null) { - return false; - } - } else if (!item2.equals(other.item2)) { - return false; - } - return true; - } - - public String getItem1() { - return item1; - } - - public String getItem2() { - return item2; + reconcileHashCode.append(mapEntry.getKey()).append(STATUS_DELIMITER).append(mapEntry.getValue().get()) + .append(STATUS_DELIMITER); } + return reconcileHashCode.toString(); } /** - * Shuffles the provided instances so that they will not always be returned in the same order. - * @param filterUpInstances whether to return only UP instances + * Shuffles the provided instances so that they will not always be returned + * in the same order. + * + * @param filterUpInstances + * whether to return only UP instances */ public void shuffleInstances(boolean filterUpInstances) { shuffleInstances(filterUpInstances, false, null, null, null); } /** - * Shuffles a whole region so that the instances will not always be returned in the same order. - * @param remoteRegionsRegistry the map of remote region names to their registries - * @param clientConfig the {@link EurekaClientConfig}, whose settings will be used to determine whether to - * filter to only UP instances - * @param instanceRegionChecker the instance region checker + * Shuffles a whole region so that the instances will not always be returned + * in the same order. + * + * @param remoteRegionsRegistry + * the map of remote region names to their registries + * @param clientConfig + * the {@link EurekaClientConfig}, whose settings will be used to + * determine whether to filter to only UP instances + * @param instanceRegionChecker + * the instance region checker */ public void shuffleAndIndexInstances(Map remoteRegionsRegistry, - EurekaClientConfig clientConfig, InstanceRegionChecker instanceRegionChecker) { + EurekaClientConfig clientConfig, InstanceRegionChecker instanceRegionChecker) { shuffleInstances(clientConfig.shouldFilterOnlyUpInstances(), true, remoteRegionsRegistry, clientConfig, instanceRegionChecker); } - private void shuffleInstances(boolean filterUpInstances, boolean indexByRemoteRegions, - @Nullable Map remoteRegionsRegistry, - @Nullable EurekaClientConfig clientConfig, - @Nullable InstanceRegionChecker instanceRegionChecker) { - this.virtualHostNameAppMap.clear(); - this.secureVirtualHostNameAppMap.clear(); + private void shuffleInstances(boolean filterUpInstances, + boolean indexByRemoteRegions, + @Nullable Map remoteRegionsRegistry, + @Nullable EurekaClientConfig clientConfig, + @Nullable InstanceRegionChecker instanceRegionChecker) { + Map secureVirtualHostNameAppMap = new HashMap<>(); + Map virtualHostNameAppMap = new HashMap<>(); for (Application application : appNameApplicationMap.values()) { if (indexByRemoteRegions) { application.shuffleAndStoreInstances(remoteRegionsRegistry, clientConfig, instanceRegionChecker); } else { application.shuffleAndStoreInstances(filterUpInstances); } - this.addInstancesToVIPMaps(application); + this.addInstancesToVIPMaps(application, virtualHostNameAppMap, secureVirtualHostNameAppMap); } - shuffleAndFilterInstances(this.virtualHostNameAppMap, - this.shuffleVirtualHostNameMap, virtualHostNameIndexMap, - filterUpInstances); - shuffleAndFilterInstances(this.secureVirtualHostNameAppMap, - this.shuffledSecureVirtualHostNameMap, - secureVirtualHostNameIndexMap, filterUpInstances); + shuffleAndFilterInstances(virtualHostNameAppMap, filterUpInstances); + shuffleAndFilterInstances(secureVirtualHostNameAppMap, filterUpInstances); + + this.virtualHostNameAppMap.putAll(virtualHostNameAppMap); + this.virtualHostNameAppMap.keySet().retainAll(virtualHostNameAppMap.keySet()); + this.secureVirtualHostNameAppMap.putAll(secureVirtualHostNameAppMap); + this.secureVirtualHostNameAppMap.keySet().retainAll(secureVirtualHostNameAppMap.keySet()); } /** @@ -461,11 +336,10 @@ private void shuffleInstances(boolean filterUpInstances, boolean indexByRemoteRe * @return AtomicLong value representing the next round-robin index. */ public AtomicLong getNextIndex(String virtualHostname, boolean secure) { - if (secure) { - return this.secureVirtualHostNameIndexMap.get(virtualHostname); - } else { - return this.virtualHostNameIndexMap.get(virtualHostname); - } + Map index = (secure) ? secureVirtualHostNameAppMap : virtualHostNameAppMap; + return Optional.ofNullable(index.get(virtualHostname.toUpperCase(Locale.ROOT))) + .map(VipIndexSupport::getRoundRobinIndex) + .orElse(null); } /** @@ -473,37 +347,23 @@ public AtomicLong getNextIndex(String virtualHostname, boolean secure) { * required. * */ - private void shuffleAndFilterInstances( - Map> srcMap, - Map>> destMap, - Map vipIndexMap, boolean filterUpInstances) { - for (Map.Entry> entries : srcMap.entrySet()) { - AbstractQueue instanceInfoQueue = entries.getValue(); - List l = new ArrayList(instanceInfoQueue); + private void shuffleAndFilterInstances(Map srcMap, boolean filterUpInstances) { + + Random shuffleRandom = new Random(); + for (Map.Entry entries : srcMap.entrySet()) { + VipIndexSupport vipIndexSupport = entries.getValue(); + AbstractQueue vipInstances = vipIndexSupport.instances; + final List filteredInstances; if (filterUpInstances) { - Iterator it = l.iterator(); - - while (it.hasNext()) { - InstanceInfo instanceInfo = it.next(); - if (!InstanceStatus.UP.equals(instanceInfo.getStatus())) { - it.remove(); - } - } - } - Collections.shuffle(l); - AtomicReference> instanceInfoList = destMap.get(entries.getKey()); - if (instanceInfoList == null) { - instanceInfoList = new AtomicReference>(l); - destMap.put(entries.getKey(), instanceInfoList); + filteredInstances = vipInstances.stream().filter(ii -> ii.getStatus() == InstanceStatus.UP) + .collect(Collectors.toCollection(() -> new ArrayList<>(vipInstances.size()))); + } else { + filteredInstances = new ArrayList(vipInstances); } - instanceInfoList.set(l); - vipIndexMap.put(entries.getKey(), new AtomicLong(0)); + Collections.shuffle(filteredInstances, shuffleRandom); + vipIndexSupport.vipList.set(filteredInstances); + vipIndexSupport.roundRobinIndex.set(0); } - - // finally remove all vips that are completed deleted (i.e. missing) from the srcSet - Set srcVips = srcMap.keySet(); - Set destVips = destMap.keySet(); - destVips.retainAll(srcVips); } /** @@ -512,39 +372,36 @@ private void shuffleAndFilterInstances( * addresses. * */ - private void addInstanceToMap(InstanceInfo info, String vipAddresses, - Map> vipMap) { + private void addInstanceToMap(InstanceInfo info, String vipAddresses, Map vipMap) { if (vipAddresses != null) { - String[] vipAddressArray = vipAddresses.split(","); + String[] vipAddressArray = vipAddresses.toUpperCase(Locale.ROOT).split(","); for (String vipAddress : vipAddressArray) { - String vipName = vipAddress.toUpperCase(Locale.ROOT); - AbstractQueue instanceInfoList = vipMap.get(vipName); - if (instanceInfoList == null) { - instanceInfoList = new ConcurrentLinkedQueue(); - vipMap.put(vipName, instanceInfoList); - } - instanceInfoList.add(info); + VipIndexSupport vis = vipMap.computeIfAbsent(vipAddress, k -> new VipIndexSupport()); + vis.instances.add(info); } } } /** * Adds the instances to the internal vip address map. - * @param app - the applications for which the instances need to be added. + * + * @param app + * - the applications for which the instances need to be added. */ - private void addInstancesToVIPMaps(Application app) { + private void addInstancesToVIPMaps(Application app, Map virtualHostNameAppMap, + Map secureVirtualHostNameAppMap) { // Check and add the instances to the their respective virtual host name // mappings for (InstanceInfo info : app.getInstances()) { String vipAddresses = info.getVIPAddress(); + if (vipAddresses != null) { + addInstanceToMap(info, vipAddresses, virtualHostNameAppMap); + } + String secureVipAddresses = info.getSecureVipAddress(); - if ((vipAddresses == null) && (secureVipAddresses == null)) { - continue; + if (secureVipAddresses != null) { + addInstanceToMap(info, secureVipAddresses, secureVirtualHostNameAppMap); } - addInstanceToMap(info, vipAddresses, virtualHostNameAppMap); - addInstanceToMap(info, secureVipAddresses, - secureVirtualHostNameAppMap); } } - } diff --git a/eureka-client/src/main/java/com/netflix/discovery/util/DeserializerStringCache.java b/eureka-client/src/main/java/com/netflix/discovery/util/DeserializerStringCache.java new file mode 100644 index 0000000000..879bd0fb5c --- /dev/null +++ b/eureka-client/src/main/java/com/netflix/discovery/util/DeserializerStringCache.java @@ -0,0 +1,463 @@ +package com.netflix.discovery.util; + +import java.io.IOException; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Map.Entry; +import java.util.PrimitiveIterator.OfInt; +import java.util.function.BiConsumer; +import java.util.function.Function; +import java.util.function.Supplier; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.ObjectReader; + +/** + * A non-locking alternative to {@link String#intern()} and {@link StringCache} + * that works with Jackson's DeserializationContext. Definitely NOT thread-safe, + * intended to avoid the costs associated with thread synchronization and + * short-lived heap allocations (e.g., Strings) + * + */ +public class DeserializerStringCache implements Function { + + public enum CacheScope { + // Strings in this scope are freed on deserialization of each + // Application element + APPLICATION_SCOPE, + // Strings in this scope are freed when overall deserialization is + // completed + GLOBAL_SCOPE + } + + private static final Logger logger = LoggerFactory.getLogger(DeserializerStringCache.class); + private static final boolean logEnabled = logger.isTraceEnabled(); + private static final String ATTR_STRING_CACHE = "deserInternCache"; + private static final int LENGTH_LIMIT = 256; + private static final int LRU_LIMIT = 1024 * 40; + + private final Map globalCache; + private final Map applicationCache; + private final int lengthLimit = LENGTH_LIMIT; + + /** + * adds a new DeserializerStringCache to the passed-in ObjectReader + * + * @param reader + * @return a wrapped ObjectReader with the string cache attribute + */ + public static ObjectReader init(ObjectReader reader) { + return reader.withAttribute(ATTR_STRING_CACHE, new DeserializerStringCache( + new HashMap(2048), new LinkedHashMap(4096, 0.75f, true) { + @Override + protected boolean removeEldestEntry(Entry eldest) { + return size() > LRU_LIMIT; + } + + })); + } + + /** + * adds an existing DeserializerStringCache from the DeserializationContext + * to an ObjectReader + * + * @param reader + * a new ObjectReader + * @param context + * an existing DeserializationContext containing a + * DeserializerStringCache + * @return a wrapped ObjectReader with the string cache attribute + */ + public static ObjectReader init(ObjectReader reader, DeserializationContext context) { + return withCache(context, cache -> { + if (cache == null) + throw new IllegalStateException(); + return reader.withAttribute(ATTR_STRING_CACHE, cache); + }); + } + + /** + * extracts a DeserializerStringCache from the DeserializationContext + * + * @param context + * an existing DeserializationContext containing a + * DeserializerStringCache + * @return a wrapped ObjectReader with the string cache attribute + */ + public static DeserializerStringCache from(DeserializationContext context) { + return withCache(context, cache -> { + if (cache == null) { + cache = new DeserializerStringCache(new HashMap(), + new HashMap()); + } + return cache; + }); + } + + /** + * clears app-scoped cache entries from the specified ObjectReader + * + * @param reader + */ + public static void clear(ObjectReader reader) { + clear(reader, CacheScope.APPLICATION_SCOPE); + } + + /** + * clears cache entries in the given scope from the specified ObjectReader. + * Always clears app-scoped entries. + * + * @param reader + * @param scope + */ + public static void clear(ObjectReader reader, final CacheScope scope) { + withCache(reader, cache -> { + if (scope == CacheScope.GLOBAL_SCOPE) { + if (logEnabled) + logger.debug("clearing global-level cache with size {}", cache.globalCache.size()); + cache.globalCache.clear(); + } + if (logEnabled) + logger.debug("clearing app-level serialization cache with size {}", cache.applicationCache.size()); + cache.applicationCache.clear(); + return null; + }); + } + + /** + * clears app-scoped cache entries from the specified DeserializationContext + * + * @param context + */ + public static void clear(DeserializationContext context) { + clear(context, CacheScope.APPLICATION_SCOPE); + } + + /** + * clears cache entries in the given scope from the specified + * DeserializationContext. Always clears app-scoped entries. + * + * @param context + * @param scope + */ + public static void clear(DeserializationContext context, CacheScope scope) { + withCache(context, cache -> { + if (scope == CacheScope.GLOBAL_SCOPE) { + if (logEnabled) + logger.debug("clearing global-level serialization cache", cache.globalCache.size()); + cache.globalCache.clear(); + } + if (logEnabled) + logger.debug("clearing app-level serialization cache with size {}", cache.applicationCache.size()); + cache.applicationCache.clear(); + return null; + }); + } + + private static T withCache(DeserializationContext context, Function consumer) { + DeserializerStringCache cache = (DeserializerStringCache) context.getAttribute(ATTR_STRING_CACHE); + return consumer.apply(cache); + } + + private static T withCache(ObjectReader reader, Function consumer) { + DeserializerStringCache cache = (DeserializerStringCache) reader.getAttributes() + .getAttribute(ATTR_STRING_CACHE); + return consumer.apply(cache); + } + + private DeserializerStringCache(Map globalCache, Map applicationCache) { + this.globalCache = globalCache; + this.applicationCache = applicationCache; + } + + public ObjectReader initReader(ObjectReader reader) { + return reader.withAttribute(ATTR_STRING_CACHE, this); + } + + /** + * returns a String read from the JsonParser argument's current position. + * The returned value may be interned at the app scope to reduce heap + * consumption + * + * @param jp + * @return a possibly interned String + * @throws IOException + */ + public String apply(final JsonParser jp) throws IOException { + return apply(jp, CacheScope.APPLICATION_SCOPE, null); + } + + public String apply(final JsonParser jp, CacheScope cacheScope) throws IOException { + return apply(jp, cacheScope, null); + } + + /** + * returns a String read from the JsonParser argument's current position. + * The returned value may be interned at the given cacheScope to reduce heap + * consumption + * + * @param jp + * @param cacheScope + * @return a possibly interned String + * @throws IOException + */ + public String apply(final JsonParser jp, CacheScope cacheScope, Supplier source) throws IOException { + return apply(CharBuffer.wrap(jp, source), cacheScope); + } + + /** + * returns a String that may be interned at app-scope to reduce heap + * consumption + * + * @param charValue + * @return a possibly interned String + */ + public String apply(final CharBuffer charValue) { + return apply(charValue, CacheScope.APPLICATION_SCOPE); + } + + /** + * returns a object of type T that may be interned at the specified scope to + * reduce heap consumption + * + * @param charValue + * @param cacheScope + * @param trabsform + * @return a possibly interned instance of T + */ + public String apply(CharBuffer charValue, CacheScope cacheScope) { + int keyLength = charValue.length(); + if ((lengthLimit < 0 || keyLength <= lengthLimit)) { + Map cache = (cacheScope == CacheScope.GLOBAL_SCOPE) ? globalCache : applicationCache; + String value = cache.get(charValue); + if (value == null) { + value = charValue.consume((k, v) -> { + cache.put(k, v); + }); + } else { + // System.out.println("cache hit"); + } + return value; + } + return charValue.toString(); + } + + /** + * returns a String that may be interned at the app-scope to reduce heap + * consumption + * + * @param stringValue + * @return a possibly interned String + */ + @Override + public String apply(final String stringValue) { + return apply(stringValue, CacheScope.APPLICATION_SCOPE); + } + + /** + * returns a String that may be interned at the given scope to reduce heap + * consumption + * + * @param stringValue + * @param cacheScope + * @return a possibly interned String + */ + public String apply(final String stringValue, CacheScope cacheScope) { + if (stringValue != null && (lengthLimit < 0 || stringValue.length() <= lengthLimit)) { + return (String) (cacheScope == CacheScope.GLOBAL_SCOPE ? globalCache : applicationCache) + .computeIfAbsent(CharBuffer.wrap(stringValue), s -> { + if (logger.isTraceEnabled()) + logger.trace(" (string) writing new interned value {} into {} cache scope", stringValue, + cacheScope); + return stringValue; + }); + } + return stringValue; + } + + public int size() { + return globalCache.size() + applicationCache.size(); + } + + private interface CharBuffer { + + public static CharBuffer wrap(JsonParser source, Supplier stringSource) throws IOException { + return new ArrayCharBuffer(source, stringSource); + } + + public static CharBuffer wrap(JsonParser source) throws IOException { + return new ArrayCharBuffer(source); + } + + public static CharBuffer wrap(String source) { + return new StringCharBuffer(source); + } + + String consume(BiConsumer valueConsumer); + + int length(); + + OfInt chars(); + + static class ArrayCharBuffer implements CharBuffer { + private char[] source; + private int offset; + private final int length; + private int hash; + private Supplier valueTransform; + + ArrayCharBuffer(JsonParser source) throws IOException { + this.source = source.getTextCharacters(); + this.offset = source.getTextOffset(); + this.length = source.getTextLength(); + } + + ArrayCharBuffer(JsonParser source, Supplier valueTransform) throws IOException { + this(source); + this.valueTransform = valueTransform; + } + + @Override + public int length() { + return length; + } + + @Override + public int hashCode() { + if (hash == 0 && length != 0) { + hash = arrayHash(source, offset, length); + } + return hash; + } + + @Override + public boolean equals(Object other) { + if (other instanceof CharBuffer) { + CharBuffer otherBuffer = (CharBuffer) other; + if (otherBuffer.length() == length) { + OfInt otherText = otherBuffer.chars(); + for (int i = offset; i < length; i++) { + if (source[i] != otherText.nextInt()) { + return false; + } + } + return true; + } + } + return false; + } + + @Override + public OfInt chars() { + return new OfInt() { + int index = offset; + int limit = index + length; + + @Override + public boolean hasNext() { + return index < limit; + } + + @Override + public int nextInt() { + return source[index++]; + } + }; + } + + @Override + public String toString() { + return new String(this.source, offset, length); + } + + @Override + public String consume(BiConsumer valueConsumer) { + String key = new String(this.source, offset, length); + String value = valueTransform == null ? key : valueTransform.get(); + valueConsumer.accept(new StringCharBuffer(key), value); + return value; + } + + private static int arrayHash(char[] a, int offset, int length) { + if (a == null) + return 0; + int result = 0; + int limit = offset + length; + for (int i = offset; i < limit; i++) { + result = 31 * result + a[i]; + } + return result; + } + } + + static class StringCharBuffer implements CharBuffer { + private String source; + + StringCharBuffer(String source) { + this.source = source; + } + + @Override + public int hashCode() { + return source.hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other instanceof CharBuffer) { + CharBuffer otherBuffer = (CharBuffer) other; + int length = source.length(); + if (otherBuffer.length() == length) { + OfInt otherText = otherBuffer.chars(); + for (int i = 0; i < length; i++) { + if (source.charAt(i) != otherText.nextInt()) { + return false; + } + } + return true; + } + } + return false; + } + + @Override + public int length() { + return source.length(); + } + + @Override + public String toString() { + return source; + } + + @Override + public OfInt chars() { + return new OfInt() { + int index; + + @Override + public boolean hasNext() { + return index < source.length(); + } + + @Override + public int nextInt() { + return source.charAt(index++); + } + }; + } + + @Override + public String consume(BiConsumer valueConsumer) { + valueConsumer.accept(this, source); + return source; + } + } + + } +} diff --git a/eureka-client/src/main/java/com/netflix/discovery/util/EurekaEntityComparators.java b/eureka-client/src/main/java/com/netflix/discovery/util/EurekaEntityComparators.java index 42fe0c2fe9..799d05f452 100644 --- a/eureka-client/src/main/java/com/netflix/discovery/util/EurekaEntityComparators.java +++ b/eureka-client/src/main/java/com/netflix/discovery/util/EurekaEntityComparators.java @@ -43,8 +43,7 @@ public static boolean equal(AmazonInfo first, AmazonInfo second) { if (first == null || first == null && second != null) { return false; } - - return first.getMetadata().equals(second.getMetadata()); + return equal(first.getMetadata(), second.getMetadata()); } public static boolean subsetOf(DataCenterInfo first, DataCenterInfo second) { diff --git a/eureka-client/src/test/java/com/netflix/discovery/converters/EnumLookupTest.java b/eureka-client/src/test/java/com/netflix/discovery/converters/EnumLookupTest.java new file mode 100644 index 0000000000..2615642148 --- /dev/null +++ b/eureka-client/src/test/java/com/netflix/discovery/converters/EnumLookupTest.java @@ -0,0 +1,26 @@ +package com.netflix.discovery.converters; + +import org.junit.Assert; +import org.junit.Test; + +public class EnumLookupTest { + + enum TestEnum { + VAL_ONE("one"), VAL_TWO("two"), VAL_THREE("three"); + private final String name; + + private TestEnum(String name) { + this.name = name; + } + } + + @Test + public void testLookup() { + EnumLookup lookup = new EnumLookup<>(TestEnum.class, v->v.name.toCharArray()); + char[] buffer = "zeroonetwothreefour".toCharArray(); + Assert.assertSame(TestEnum.VAL_ONE, lookup.find(buffer, 4, 3)); + Assert.assertSame(TestEnum.VAL_TWO, lookup.find(buffer, 7, 3)); + Assert.assertSame(TestEnum.VAL_THREE, lookup.find(buffer, 10, 5)); + } + +} diff --git a/eureka-client/src/test/java/com/netflix/discovery/converters/EurekaJacksonCodecIntegrationTest.java b/eureka-client/src/test/java/com/netflix/discovery/converters/EurekaJacksonCodecIntegrationTest.java new file mode 100644 index 0000000000..6752181612 --- /dev/null +++ b/eureka-client/src/test/java/com/netflix/discovery/converters/EurekaJacksonCodecIntegrationTest.java @@ -0,0 +1,117 @@ +package com.netflix.discovery.converters; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.file.StandardCopyOption; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import org.junit.Test; + +import com.netflix.discovery.shared.Applications; + +/** + * this integration test parses the response of a Eureka discovery server, + * specified by url via system property 'discovery.url'. It's useful for memory + * utilization and performance tests, but since it's environment specific, the + * tests below are @Ignore'd. + * + */ +@org.junit.Ignore +public class EurekaJacksonCodecIntegrationTest { + private static final int UNREASONABLE_TIMEOUT_MS = 500; + private final EurekaJacksonCodec codec = new EurekaJacksonCodec("", ""); + + /** + * parse discovery response in a long-running loop with a delay + * + * @throws Exception + */ + @Test + public void testRealDecode() throws Exception { + Applications applications; + File localDiscovery = new File("/var/folders/6j/qy6n1npj11x5j2j_9ng2wzmw0000gp/T/discovery-data-6054758555577530004.json"); //downloadRegistration(System.getProperty("discovery.url")); + long testStart = System.currentTimeMillis(); + for (int i = 0; i < 60; i++) { + try (InputStream is = new FileInputStream(localDiscovery)) { + long start = System.currentTimeMillis(); + applications = codec.readValue(Applications.class, is); + System.out.println("found some applications: " + applications.getRegisteredApplications().size() + + " et: " + (System.currentTimeMillis() - start)); + } + } + System.out.println("test time: " + + " et: " + (System.currentTimeMillis() - testStart)); + } + + + @Test + public void testCuriosity() { + char[] arr1 = "test".toCharArray(); + char[] arr2 = new char[] {'t', 'e', 's', 't'}; + + System.out.println("array equals" + arr1.equals(arr2)); + } + + /** + * parse discovery response with an unreasonable timeout, so that the + * parsing job is cancelled + * + * @throws Exception + */ + @Test + public void testDecodeTimeout() throws Exception { + ExecutorService executor = Executors.newFixedThreadPool(5); + File localDiscovery = downloadRegistration(System.getProperty("discovery.url")); + Callable task = () -> { + try (InputStream is = new FileInputStream(localDiscovery)) { + return codec.readValue(Applications.class, is); + } + }; + + final int cancelAllButNthTask = 3; + for (int i = 0; i < 30; i++) { + Future appsFuture = executor.submit(task); + if (i % cancelAllButNthTask < cancelAllButNthTask - 1) { + Thread.sleep(UNREASONABLE_TIMEOUT_MS); + System.out.println("cancelling..." + " i: " + i + " - " + (i % 3)); + appsFuture.cancel(true); + } + try { + Applications apps = appsFuture.get(); + System.out.println("found some applications: " + apps.toString() + ":" + + apps.getRegisteredApplications().size() + " i: " + i + " - " + (i % 3)); + } catch (Exception e) { + System.out.println(e + " cause: " + " i: " + i + " - " + (i % 3)); + } + } + } + + /** + * low-tech http downloader + */ + private static File downloadRegistration(String discoveryUrl) throws IOException { + if (discoveryUrl == null) { + throw new IllegalArgumentException("null value not allowed for parameter discoveryUrl"); + } + File localFile = File.createTempFile("discovery-data-", ".json"); + URL url = new URL(discoveryUrl); + System.out.println("downloading registration data from " + url + " to " + localFile); + HttpURLConnection hurlConn = (HttpURLConnection) url.openConnection(); + hurlConn.setDoOutput(true); + hurlConn.setRequestProperty("accept", "application/json"); + hurlConn.connect(); + try (InputStream is = hurlConn.getInputStream()) { + java.nio.file.Files.copy(is, localFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + } + return localFile; + + } + +} diff --git a/eureka-client/src/test/java/com/netflix/discovery/converters/EurekaJacksonCodecTest.java b/eureka-client/src/test/java/com/netflix/discovery/converters/EurekaJacksonCodecTest.java index ea0300a165..9c45eca1f1 100644 --- a/eureka-client/src/test/java/com/netflix/discovery/converters/EurekaJacksonCodecTest.java +++ b/eureka-client/src/test/java/com/netflix/discovery/converters/EurekaJacksonCodecTest.java @@ -1,21 +1,23 @@ package com.netflix.discovery.converters; -import javax.ws.rs.core.MediaType; +import static org.junit.Assert.assertTrue; + import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.InputStream; import java.nio.charset.Charset; import java.util.Iterator; +import javax.ws.rs.core.MediaType; + +import org.junit.Test; + import com.netflix.appinfo.InstanceInfo; import com.netflix.appinfo.InstanceInfo.ActionType; import com.netflix.discovery.shared.Application; import com.netflix.discovery.shared.Applications; import com.netflix.discovery.util.EurekaEntityComparators; import com.netflix.discovery.util.InstanceInfoGenerator; -import org.junit.Test; - -import static org.junit.Assert.assertTrue; /** * @author Tomasz Bak @@ -183,7 +185,7 @@ public void testApplicationsXStreamEncodeJacksonDecode() throws Exception { ByteArrayOutputStream captureStream = new ByteArrayOutputStream(); new EntityBodyConverter().write(original, captureStream, MediaType.APPLICATION_JSON_TYPE); byte[] encoded = captureStream.toByteArray(); - + String encodedString = new String(encoded); // Decode InputStream source = new ByteArrayInputStream(encoded); Applications decoded = codec.readValue(Applications.class, source); @@ -226,4 +228,5 @@ public void testJacksonWrite() throws Exception { assertTrue(EurekaEntityComparators.equal(decoded, INSTANCE_INFO_1_A1)); } + } \ No newline at end of file diff --git a/eureka-client/src/test/java/com/netflix/discovery/shared/ApplicationsTest.java b/eureka-client/src/test/java/com/netflix/discovery/shared/ApplicationsTest.java index 93d5fd2ed7..bc09becc94 100644 --- a/eureka-client/src/test/java/com/netflix/discovery/shared/ApplicationsTest.java +++ b/eureka-client/src/test/java/com/netflix/discovery/shared/ApplicationsTest.java @@ -1,23 +1,56 @@ package com.netflix.discovery.shared; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import java.lang.reflect.Constructor; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; import com.google.common.collect.Iterables; +import com.netflix.appinfo.AmazonInfo; import com.netflix.appinfo.DataCenterInfo; import com.netflix.appinfo.InstanceInfo; -import org.junit.Test; - -import static junit.framework.Assert.assertNull; -import static junit.framework.Assert.assertTrue; -import static org.junit.Assert.assertEquals; +import com.netflix.appinfo.AmazonInfo.MetaDataKey; +import com.netflix.appinfo.InstanceInfo.InstanceStatus; +import com.netflix.discovery.AzToRegionMapper; +import com.netflix.discovery.DefaultEurekaClientConfig; +import com.netflix.discovery.EurekaClientConfig; +import com.netflix.discovery.InstanceRegionChecker; +import com.netflix.discovery.InstanceRegionCheckerTest; +import com.netflix.discovery.PropertyBasedAzToRegionMapper; public class ApplicationsTest { + @Test + public void testVersionAndAppHash() { + Applications apps = new Applications(); + assertEquals(-1L, (long)apps.getVersion()); + assertNull(apps.getAppsHashCode()); + + apps.setVersion(101L); + apps.setAppsHashCode("UP_5_DOWN_6_"); + assertEquals(101L, (long)apps.getVersion()); + assertEquals("UP_5_DOWN_6_", apps.getAppsHashCode()); + } + /** - * Test that instancesMap in Application and shuffleVirtualHostNameMap in Applications are - * correctly updated when the last instance is removed from an application and shuffleInstances - * has been run. + * Test that instancesMap in Application and shuffleVirtualHostNameMap in + * Applications are correctly updated when the last instance is removed from + * an application and shuffleInstances has been run. */ @Test public void shuffleVirtualHostNameMapLastInstanceTest() { @@ -26,27 +59,302 @@ public DataCenterInfo.Name getName() { return DataCenterInfo.Name.MyOwn; } }; - InstanceInfo instanceInfo = InstanceInfo.Builder.newBuilder() - .setAppName("test") - .setVIPAddress("test.testname:1") - .setDataCenterInfo(myDCI) - .setHostName("test.hostname").build(); + InstanceInfo instanceInfo = InstanceInfo.Builder.newBuilder().setAppName("test") + .setVIPAddress("test.testname:1").setDataCenterInfo(myDCI).setHostName("test.hostname").build(); Application application = new Application("TestApp"); application.addInstance(instanceInfo); Applications applications = new Applications(); applications.addApplication(application); applications.shuffleInstances(true); - List testApp = applications.getInstancesByVirtualHostName("test.testname:1"); - assertEquals(Iterables.getOnlyElement(testApp), - application.getByInstanceId("test.hostname")); + List testApp = applications.getInstancesByVirtualHostName("test.testname:1"); + assertEquals(Iterables.getOnlyElement(testApp), application.getByInstanceId("test.hostname")); application.removeInstance(instanceInfo); + assertEquals(0, applications.size()); + applications.shuffleInstances(true); testApp = applications.getInstancesByVirtualHostName("test.testname:1"); + assertTrue(testApp.isEmpty()); assertNull(application.getByInstanceId("test.hostname")); + } + + /** + * Test that instancesMap in Application and shuffleVirtualHostNameMap in + * Applications are correctly updated when the last instance is removed from + * an application and shuffleInstances has been run. + */ + @Test + public void shuffleSecureVirtualHostNameMapLastInstanceTest() { + DataCenterInfo myDCI = new DataCenterInfo() { + public DataCenterInfo.Name getName() { + return DataCenterInfo.Name.MyOwn; + } + }; + InstanceInfo instanceInfo = InstanceInfo.Builder.newBuilder().setAppName("test") + .setVIPAddress("test.testname:1").setSecureVIPAddress("securetest.testname:7102") + .setDataCenterInfo(myDCI).setHostName("test.hostname").build(); + + Application application = new Application("TestApp"); + application.addInstance(instanceInfo); + + Applications applications = new Applications(); + assertEquals(0, applications.size()); + + applications.addApplication(application); + assertEquals(1, applications.size()); + + applications.shuffleInstances(true); + List testApp = applications.getInstancesByVirtualHostName("test.testname:1"); + + assertEquals(Iterables.getOnlyElement(testApp), application.getByInstanceId("test.hostname")); + + application.removeInstance(instanceInfo); + assertNull(application.getByInstanceId("test.hostname")); + assertEquals(0, applications.size()); + + applications.shuffleInstances(true); + testApp = applications.getInstancesBySecureVirtualHostName("securetest.testname:7102"); assertTrue(testApp.isEmpty()); + + assertNull(application.getByInstanceId("test.hostname")); } + + /** + * Test that instancesMap in Application and shuffleVirtualHostNameMap in + * Applications are correctly updated when the last instance is removed from + * an application and shuffleInstances has been run. + */ + @Test + public void shuffleRemoteRegistryTest() throws Exception { + AmazonInfo ai1 = AmazonInfo.Builder.newBuilder() + .addMetadata(MetaDataKey.availabilityZone, "us-east-1a") + .build(); + InstanceInfo instanceInfo1 = InstanceInfo.Builder.newBuilder().setAppName("test") + .setVIPAddress("test.testname:1") + .setSecureVIPAddress("securetest.testname:7102") + .setDataCenterInfo(ai1) + .setAppName("TestApp") + .setHostName("test.east.hostname") + .build(); + AmazonInfo ai2 = AmazonInfo.Builder.newBuilder() + .addMetadata(MetaDataKey.availabilityZone, "us-west-2a") + .build(); + InstanceInfo instanceInfo2 = InstanceInfo.Builder.newBuilder().setAppName("test") + .setVIPAddress("test.testname:1") + .setSecureVIPAddress("securetest.testname:7102") + .setDataCenterInfo(ai2) + .setAppName("TestApp") + .setHostName("test.west.hostname") + .build(); + + Application application = new Application("TestApp"); + application.addInstance(instanceInfo1); + application.addInstance(instanceInfo2); + + Applications applications = new Applications(); + assertEquals(0, applications.size()); + + applications.addApplication(application); + assertEquals(2, applications.size()); + + EurekaClientConfig clientConfig = Mockito.mock(EurekaClientConfig.class); + Mockito.when(clientConfig.getAvailabilityZones("us-east-1")).thenReturn(new String[] {"us-east-1a", "us-east-1b", "us-east-1c", "us-east-1d", "us-east-1e", "us-east-1f"}); + Mockito.when(clientConfig.getAvailabilityZones("us-west-2")).thenReturn(new String[] {"us-west-2a", "us-west-2b", "us-west-2c"}); + Mockito.when(clientConfig.getRegion()).thenReturn("us-east-1"); + Constructor ctor = InstanceRegionChecker.class.getDeclaredConstructor(AzToRegionMapper.class, String.class); + ctor.setAccessible(true); + PropertyBasedAzToRegionMapper azToRegionMapper = new PropertyBasedAzToRegionMapper(clientConfig); + azToRegionMapper.setRegionsToFetch(new String[] {"us-east-1", "us-west-2"}); + InstanceRegionChecker instanceRegionChecker = (InstanceRegionChecker)ctor.newInstance(azToRegionMapper, "us-west-2"); + Map remoteRegionsRegistry = new HashMap<>(); + remoteRegionsRegistry.put("us-east-1", new Applications()); + applications.shuffleAndIndexInstances(remoteRegionsRegistry, clientConfig, instanceRegionChecker); + assertNotNull(remoteRegionsRegistry.get("us-east-1").getRegisteredApplications("TestApp").getByInstanceId("test.east.hostname")); + assertNull(applications.getRegisteredApplications("TestApp").getByInstanceId("test.east.hostname")); + assertNull(remoteRegionsRegistry.get("us-east-1").getRegisteredApplications("TestApp").getByInstanceId("test.west.hostname")); + assertNotNull(applications.getRegisteredApplications("TestApp").getByInstanceId("test.west.hostname")); + + } + + + @Test + public void testRegisteredApplications() { + DataCenterInfo myDCI = new DataCenterInfo() { + public DataCenterInfo.Name getName() { + return DataCenterInfo.Name.MyOwn; + } + }; + InstanceInfo instanceInfo = InstanceInfo.Builder.newBuilder() + .setAppName("test") + .setVIPAddress("test.testname:1") + .setSecureVIPAddress("securetest.testname:7102") + .setDataCenterInfo(myDCI) + .setHostName("test.hostname") + .build(); + + Application application = new Application("TestApp"); + application.addInstance(instanceInfo); + + Applications applications = new Applications(); + applications.addApplication(application); + + List appsList = applications.getRegisteredApplications(); + Assert.assertEquals(1, appsList.size()); + Assert.assertTrue(appsList.contains(application)); + Assert.assertEquals(application, applications.getRegisteredApplications(application.getName())); + } + + @Test + public void testRegisteredApplicationsConstructor() { + DataCenterInfo myDCI = new DataCenterInfo() { + public DataCenterInfo.Name getName() { + return DataCenterInfo.Name.MyOwn; + } + }; + InstanceInfo instanceInfo = InstanceInfo.Builder.newBuilder() + .setAppName("test") + .setVIPAddress("test.testname:1") + .setSecureVIPAddress("securetest.testname:7102") + .setDataCenterInfo(myDCI) + .setHostName("test.hostname") + .build(); + + Application application = new Application("TestApp"); + application.addInstance(instanceInfo); + + Applications applications = new Applications("UP_1_", -1L, Arrays.asList(application)); + + List appsList = applications.getRegisteredApplications(); + Assert.assertEquals(1, appsList.size()); + Assert.assertTrue(appsList.contains(application)); + Assert.assertEquals(application, applications.getRegisteredApplications(application.getName())); + } + + @Test + public void testApplicationsHashAndVersion() { + Applications applications = new Applications("appsHashCode", 1L, Collections.emptyList()); + assertEquals(1L, (long)applications.getVersion()); + assertEquals("appsHashCode", applications.getAppsHashCode()); + } + + @Test + public void testPopulateInstanceCount() { + DataCenterInfo myDCI = new DataCenterInfo() { + public DataCenterInfo.Name getName() { + return DataCenterInfo.Name.MyOwn; + } + }; + InstanceInfo instanceInfo = InstanceInfo.Builder.newBuilder() + .setAppName("test") + .setVIPAddress("test.testname:1") + .setSecureVIPAddress("securetest.testname:7102") + .setDataCenterInfo(myDCI) + .setHostName("test.hostname") + .setStatus(InstanceStatus.UP) + .build(); + + Application application = new Application("TestApp"); + application.addInstance(instanceInfo); + + Applications applications = new Applications(); + applications.addApplication(application); + + TreeMap instanceCountMap = new TreeMap<>(); + applications.populateInstanceCountMap(instanceCountMap); + assertEquals(1, instanceCountMap.size()); + assertNotNull(instanceCountMap.get(InstanceStatus.UP.name())); + assertEquals(1, instanceCountMap.get(InstanceStatus.UP.name()).get()); + + } + + @Test + public void testGetNextIndex() { + DataCenterInfo myDCI = new DataCenterInfo() { + public DataCenterInfo.Name getName() { + return DataCenterInfo.Name.MyOwn; + } + }; + InstanceInfo instanceInfo = InstanceInfo.Builder.newBuilder() + .setAppName("test") + .setVIPAddress("test.testname:1") + .setSecureVIPAddress("securetest.testname:7102") + .setDataCenterInfo(myDCI) + .setHostName("test.hostname") + .setStatus(InstanceStatus.UP) + .build(); + + Application application = new Application("TestApp"); + application.addInstance(instanceInfo); + + Applications applications = new Applications(); + applications.addApplication(application); + + assertNotNull(applications.getNextIndex("test.testname:1", false)); + assertEquals(0L, applications.getNextIndex("test.testname:1", false).get()); + assertNotNull(applications.getNextIndex("securetest.testname:7102", true)); + assertEquals(0L, applications.getNextIndex("securetest.testname:7102", true).get()); + assertNotSame(applications.getNextIndex("test.testname:1", false), applications.getNextIndex("securetest.testname:7102", true)); + } + + @Test + public void testReconcileHashcode() { + DataCenterInfo myDCI = new DataCenterInfo() { + public DataCenterInfo.Name getName() { + return DataCenterInfo.Name.MyOwn; + } + }; + InstanceInfo instanceInfo = InstanceInfo.Builder.newBuilder() + .setAppName("test") + .setVIPAddress("test.testname:1") + .setSecureVIPAddress("securetest.testname:7102") + .setDataCenterInfo(myDCI) + .setHostName("test.hostname") + .setStatus(InstanceStatus.UP) + .build(); + + Application application = new Application("TestApp"); + application.addInstance(instanceInfo); + + Applications applications = new Applications(); + + String hashCode = applications.getReconcileHashCode(); + assertTrue(hashCode.isEmpty()); + + applications.addApplication(application); + hashCode = applications.getReconcileHashCode(); + assertFalse(hashCode.isEmpty()); + assertEquals("UP_1_", hashCode); + } + + @Test + public void testInstanceFiltering() { + DataCenterInfo myDCI = new DataCenterInfo() { + public DataCenterInfo.Name getName() { + return DataCenterInfo.Name.MyOwn; + } + }; + InstanceInfo instanceInfo = InstanceInfo.Builder.newBuilder() + .setAppName("test") + .setVIPAddress("test.testname:1") + .setSecureVIPAddress("securetest.testname:7102") + .setDataCenterInfo(myDCI) + .setHostName("test.hostname") + .setStatus(InstanceStatus.DOWN) + .build(); + + Application application = new Application("TestApp"); + application.addInstance(instanceInfo); + + Applications applications = new Applications(); + applications.addApplication(application); + applications.shuffleInstances(true); + + assertNotNull(applications.getRegisteredApplications("TestApp").getByInstanceId("test.hostname")); + assertTrue(applications.getInstancesBySecureVirtualHostName("securetest.testname:7102").isEmpty()); + assertTrue(applications.getInstancesBySecureVirtualHostName("test.testname:1").isEmpty()); + } + } diff --git a/eureka-core/src/main/java/com/netflix/eureka/registry/RemoteRegionRegistry.java b/eureka-core/src/main/java/com/netflix/eureka/registry/RemoteRegionRegistry.java index be7e74973c..393590ae9f 100644 --- a/eureka-core/src/main/java/com/netflix/eureka/registry/RemoteRegionRegistry.java +++ b/eureka-core/src/main/java/com/netflix/eureka/registry/RemoteRegionRegistry.java @@ -415,18 +415,6 @@ private boolean reconcileAndLogDifference(Applications delta, String reconcileHa reconcileHashCode, delta.getAppsHashCode()); Applications serverApps = this.fetchRemoteRegistry(false); - - Map> reconcileDiffMap = getApplications().getReconcileMapDiff(serverApps); - String reconcileString = ""; - for (Map.Entry> mapEntry : reconcileDiffMap - .entrySet()) { - reconcileString = reconcileString + mapEntry.getKey() + ": "; - for (String displayString : mapEntry.getValue()) { - reconcileString = reconcileString + displayString; - } - reconcileString = reconcileString + "\n"; - } - logger.warn("The reconcile string is {}", reconcileString); applications.set(serverApps); applicationsDelta.set(serverApps); logger.warn("The Reconcile hashcodes after complete sync up, client : {}, server : {}.",