Skip to content

Commit

Permalink
(feat) refactor ThreadId and replicator (sofastack#169)
Browse files Browse the repository at this point in the history
* (feat) refactor ThreadId and replicator

* (feat) Adds javadoc
  • Loading branch information
killme2008 authored and fengjiachun committed May 22, 2019
1 parent b76e3e6 commit d6ac5c2
Show file tree
Hide file tree
Showing 4 changed files with 117 additions and 108 deletions.
85 changes: 47 additions & 38 deletions jraft-core/src/main/java/com/alipay/sofa/jraft/core/Replicator.java
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,8 @@ private ReplicatorMetricSet(final ReplicatorOptions opts, final Replicator r) {
@Override
public Map<String, Metric> getMetrics() {
final Map<String, Metric> gauges = new HashMap<>();
gauges.put("log-lags", (Gauge<Long>) () -> this.opts.getLogManager().getLastLogIndex() - (this.r.nextIndex - 1));
gauges.put("log-lags",
(Gauge<Long>) () -> this.opts.getLogManager().getLastLogIndex() - (this.r.nextIndex - 1));
gauges.put("next-index", (Gauge<Long>) () -> this.r.nextIndex);
gauges.put("heartbeat-times", (Gauge<Long>) () -> this.r.heartbeatCounter);
gauges.put("install-snapshot-times", (Gauge<Long>) () -> this.r.installSnapshotCounter);
Expand Down Expand Up @@ -467,6 +468,7 @@ void installSnapshot() {
final NodeImpl node = this.options.getNode();
final RaftException error = new RaftException(EnumOutter.ErrorType.ERROR_TYPE_SNAPSHOT);
error.setStatus(new Status(RaftError.EIO, "Fail to generate uri for snapshot reader"));
releaseReader();
this.id.unlock();
doUnlock = false;
node.onError(error);
Expand All @@ -478,6 +480,7 @@ void installSnapshot() {
final NodeImpl node = this.options.getNode();
final RaftException error = new RaftException(EnumOutter.ErrorType.ERROR_TYPE_SNAPSHOT);
error.setStatus(new Status(RaftError.EIO, "Fail to load meta from %s", snapshotPath));
releaseReader();
this.id.unlock();
doUnlock = false;
node.onError(error);
Expand Down Expand Up @@ -524,10 +527,7 @@ static boolean onInstallSnapshotReturned(final ThreadId id, final Replicator r,
final InstallSnapshotRequest request,
final InstallSnapshotResponse response) {
boolean success = true;
if (r.reader != null) {
Utils.closeQuietly(r.reader);
r.reader = null;
}
r.releaseReader();
// noinspection ConstantConditions
do {
final StringBuilder sb = new StringBuilder("Node "). //
Expand Down Expand Up @@ -725,7 +725,8 @@ private static String getReplicatorMetricName(final ReplicatorOptions opts) {
return "replicator-" + opts.getNode().getGroupId() + "/" + opts.getPeerId();
}

public static void waitForCaughtUp(final ThreadId id, final long maxMargin, final long dueTime, final CatchUpClosure done) {
public static void waitForCaughtUp(final ThreadId id, final long maxMargin, final long dueTime,
final CatchUpClosure done) {
final Replicator r = (Replicator) id.lock();

if (r == null) {
Expand Down Expand Up @@ -838,32 +839,39 @@ void block(final long startTimeMs, @SuppressWarnings("unused") final int errorCo
public void onError(final ThreadId id, final Object data, final int errorCode) {
final Replicator r = (Replicator) data;
if (errorCode == RaftError.ESTOP.getNumber()) {
for (final Inflight inflight : r.inflights) {
if (inflight != r.rpcInFly) {
inflight.rpcFuture.cancel(true);
try {
for (final Inflight inflight : r.inflights) {
if (inflight != r.rpcInFly) {
inflight.rpcFuture.cancel(true);
}
}
if (r.rpcInFly != null) {
r.rpcInFly.rpcFuture.cancel(true);
r.rpcInFly = null;
}
if (r.heartbeatInFly != null) {
r.heartbeatInFly.cancel(true);
r.heartbeatInFly = null;
}
if (r.timeoutNowInFly != null) {
r.timeoutNowInFly.cancel(true);
r.timeoutNowInFly = null;
}
if (r.heartbeatTimer != null) {
r.heartbeatTimer.cancel(true);
r.heartbeatTimer = null;
}
if (r.blockTimer != null) {
r.blockTimer.cancel(true);
r.blockTimer = null;
}
if (r.waitId >= 0) {
r.options.getLogManager().removeWaiter(r.waitId);
}
r.notifyOnCaughtUp(errorCode, true);
} finally {
r.destroy();
}
if (r.rpcInFly != null) {
r.rpcInFly.rpcFuture.cancel(true);
r.rpcInFly = null;
}
if (r.heartbeatInFly != null) {
r.heartbeatInFly.cancel(true);
r.heartbeatInFly = null;
}
if (r.timeoutNowInFly != null) {
r.timeoutNowInFly.cancel(true);
r.timeoutNowInFly = null;
}
if (r.heartbeatTimer != null) {
r.heartbeatTimer.cancel(true);
r.heartbeatTimer = null;
}
if (r.waitId >= 0) {
r.options.getLogManager().removeWaiter(r.waitId);
}
r.notifyOnCaughtUp(errorCode, true);
r.destroy();
} else if (errorCode == RaftError.ETIMEDOUT.getNumber()) {
id.unlock();
Utils.runInThread(() -> sendHeartbeat(id));
Expand Down Expand Up @@ -932,10 +940,7 @@ void destroy() {
final ThreadId savedId = this.id;
LOG.info("Replicator {} is going to quit", savedId);
this.id = null;
if (this.reader != null) {
Utils.closeQuietly(this.reader);
this.reader = null;
}
releaseReader();
// Unregister replicator metric set
if (this.options.getNode().getNodeMetrics().getMetricRegistry() != null) {
this.options.getNode().getNodeMetrics().getMetricRegistry().remove(getReplicatorMetricName(this.options));
Expand All @@ -944,6 +949,13 @@ void destroy() {
savedId.unlockAndDestroy();
}

private void releaseReader() {
if (this.reader != null) {
Utils.closeQuietly(this.reader);
this.reader = null;
}
}

static void onHeartbeatReturned(final ThreadId id, final Status status, final AppendEntriesRequest request,
final AppendEntriesResponse response, final long rpcSendTime) {
if (id == null) {
Expand Down Expand Up @@ -1129,10 +1141,7 @@ void resetInflights() {
this.pendingResponses.clear();
final int rs = Math.max(this.reqSeq, this.requiredNextSeq);
this.reqSeq = this.requiredNextSeq = rs;
if (this.reader != null) {
Utils.closeQuietly(this.reader);
this.reader = null;
}
releaseReader();
}

private static boolean onAppendEntriesReturned(final ThreadId id, final Inflight inflight, final Status status,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ public boolean tryLock() {
}

@Override
public boolean tryLock(long time, TimeUnit unit) throws InterruptedException {
public boolean tryLock(final long time, final TimeUnit unit) throws InterruptedException {
return tryAcquireNanos(1, unit.toNanos(time));
}

Expand All @@ -58,7 +58,7 @@ public boolean isHeldByCurrentThread() {
}

public Thread getOwner() {
return owner;
return this.owner;
}

@Override
Expand All @@ -67,26 +67,26 @@ public Condition newCondition() {
}

@Override
protected boolean tryAcquire(int acquires) {
protected boolean tryAcquire(final int acquires) {
if (compareAndSetState(0, 1)) {
owner = Thread.currentThread();
this.owner = Thread.currentThread();
return true;
}
return false;
}

@Override
protected boolean tryRelease(int releases) {
if (Thread.currentThread() != owner) {
throw new IllegalMonitorStateException();
protected boolean tryRelease(final int releases) {
if (Thread.currentThread() != this.owner) {
throw new IllegalMonitorStateException("Owner is " + this.owner);
}
owner = null;
this.owner = null;
setState(0);
return true;
}

@Override
protected boolean isHeldExclusively() {
return getState() != 0 && owner == Thread.currentThread();
return getState() != 0 && this.owner == Thread.currentThread();
}
}
99 changes: 49 additions & 50 deletions jraft-core/src/main/java/com/alipay/sofa/jraft/util/ThreadId.java
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import java.util.Collections;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand All @@ -34,40 +33,49 @@
*/
public class ThreadId {

private static final Logger LOG = LoggerFactory.getLogger(ThreadId.class);
private static final int TRY_LOCK_TIMEOUT_MS = 10;

private final Object data;
private volatile NonReentrantLock lock = new NonReentrantLock();
private final List<Integer> pendingErrors = Collections.synchronizedList(new ArrayList<>());
private final OnError onError;
private static final Logger LOG = LoggerFactory.getLogger(ThreadId.class);

private final Object data;
private final NonReentrantLock lock = new NonReentrantLock();
private final List<Integer> pendingErrors = Collections.synchronizedList(new ArrayList<>());
private final OnError onError;
private volatile boolean destroyed;

/**
* @author boyan ([email protected])
*
* 2018-Mar-29 11:01:54 AM
*/
public interface OnError {
/**
* Error callback,it will be called in lock, but should take care of unlocking it.
* @param id the thread id
* @param data the data
* @param errorCode the error code
*/
void onError(ThreadId id, Object data, int errorCode);
}

public ThreadId(Object data, OnError onError) {
public ThreadId(final Object data, final OnError onError) {
super();
this.data = data;
this.onError = onError;
this.destroyed = false;
}

public Object getData() {
return data;
return this.data;
}

public Object lock() {
final Lock theLock = this.lock;
if (theLock == null) {
if (this.destroyed) {
return null;
}
try {
while (!theLock.tryLock(1, TimeUnit.SECONDS)) {
if (this.lock == null) {
while (!this.lock.tryLock(TRY_LOCK_TIMEOUT_MS, TimeUnit.MILLISECONDS)) {
if (this.destroyed) {
return null;
}
}
Expand All @@ -76,55 +84,45 @@ public Object lock() {
return null;
}
// Got the lock, double checking state.
if (this.lock == null) {
if (this.destroyed) {
// should release lock
theLock.unlock();
this.lock.unlock();
return null;
}
return this.data;
}

public void unlock() {
final NonReentrantLock theLock = this.lock;
if (theLock == null) {
return;
}
if (!theLock.isHeldByCurrentThread()) {
if (!this.lock.isHeldByCurrentThread()) {
LOG.warn("Fail to unlock with {}, the lock is held by {} and current thread is {}.", this.data,
theLock.getOwner(), Thread.currentThread());
this.lock.getOwner(), Thread.currentThread());
return;
}
// calls all pending errors before unlock
boolean doUnlock = true;
try {
final List<Integer> errors;
synchronized (this.pendingErrors) {
errors = new ArrayList<>(this.pendingErrors);
this.pendingErrors.clear();
}
for (final Integer code : errors) {
// The lock will be unlocked in onError.
doUnlock = false;
if (this.onError != null) {
this.onError.onError(this, this.data, code);
}
}
} finally {
doUnlock(theLock);
}
}

private void doUnlock(final NonReentrantLock theLock) {
if (theLock != null) {
try {
theLock.unlock();
} catch (final Exception e) {
LOG.warn("Fail to unlock with {}, the lock is held by {} and current thread is {}.", this.data,
theLock.getOwner(), Thread.currentThread(), e);
if (doUnlock) {
this.lock.unlock();
}
}
}

public void join() {
while (this.lock != null) {
Thread.yield();
while (!this.destroyed) {
ThreadHelper.onSpinWait();
}
}

Expand All @@ -134,15 +132,16 @@ public String toString() {
}

public void unlockAndDestroy() {
final Lock theLock = this.lock;
this.lock = null;
if (theLock != null) {
try {
theLock.unlock();
} catch (final Exception ignored) {
// ignored
}
if (this.destroyed) {
return;
}
this.destroyed = true;
if (!this.lock.isHeldByCurrentThread()) {
LOG.warn("Fail to unlockAndDestroy with {}, the lock is held by {} and current thread is {}.", this.data,
this.lock.getOwner(), Thread.currentThread());
return;
}
this.lock.unlock();
}

/**
Expand All @@ -153,17 +152,17 @@ public void unlockAndDestroy() {
* @param errorCode error code
*/
public void setError(final int errorCode) {
final NonReentrantLock theLock = this.lock;
if (theLock == null) {
if (this.destroyed) {
return;
}
if (theLock.tryLock()) {
try {
if (this.onError != null) {
this.onError.onError(this, data, errorCode);
}
} finally {
doUnlock(theLock);
if (this.lock.tryLock()) {
if (this.destroyed) {
this.lock.unlock();
return;
}
if (this.onError != null) {
// The lock will be unlocked in onError.
this.onError.onError(this, this.data, errorCode);
}
} else {
this.pendingErrors.add(errorCode);
Expand Down
Loading

0 comments on commit d6ac5c2

Please sign in to comment.