Skip to content

Commit

Permalink
文件写入方式修改
Browse files Browse the repository at this point in the history
  • Loading branch information
monkeyWie committed Mar 28, 2018
1 parent f9d867a commit 1b84586
Show file tree
Hide file tree
Showing 7 changed files with 250 additions and 81 deletions.
124 changes: 124 additions & 0 deletions common/src/main/java/lee/study/down/io/Mmap.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
package lee.study.down.io;

import java.io.Closeable;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.nio.channels.FileChannel;
import sun.misc.Unsafe;
import sun.nio.ch.FileChannelImpl;

public class Mmap implements Closeable {

private static final Unsafe unsafe;
private static final Method mmap;
private static final Method unmmap;
private static final int BYTE_ARRAY_OFFSET;

private long addr, size;
private final String loc;

static {
try {
Field singleoneInstanceField = Unsafe.class.getDeclaredField("theUnsafe");
singleoneInstanceField.setAccessible(true);
unsafe = (Unsafe) singleoneInstanceField.get(null);

mmap = getMethod(FileChannelImpl.class, "map0", int.class, long.class, long.class);
unmmap = getMethod(FileChannelImpl.class, "unmap0", long.class, long.class);

BYTE_ARRAY_OFFSET = unsafe.arrayBaseOffset(byte[].class);
} catch (Exception e) {
throw new RuntimeException(e);
}
}

//Bundle reflection calls to get access to the given method
private static Method getMethod(Class<?> cls, String name, Class<?>... params) throws Exception {
Method m = cls.getDeclaredMethod(name, params);
m.setAccessible(true);
return m;
}

//Round to next 4096 bytes
private static long roundTo4096(long i) {
return (i + 0xfffL) & ~0xfffL;
}

//Given that the location and size have been set, map that location
//for the given length and set this.addr to the returned offset
private void mapAndSetOffset() throws IOException {
RandomAccessFile backingFile = null;
FileChannel ch = null;
try {
backingFile = new RandomAccessFile(this.loc, "rw");
ch = backingFile.getChannel();
this.addr = (long) mmap.invoke(ch, 1, 0L, this.size);
} catch (Exception e) {
throw new IOException("mmap mapAndSetOffset", e);
} finally {
if (ch != null) {
ch.close();
}
if (backingFile != null) {
backingFile.close();
}
}
}

public Mmap(final String loc, long len) throws IOException {
this.loc = loc;
this.size = roundTo4096(len);
mapAndSetOffset();
}

public Mmap(final String loc, long pos, long len) throws Exception {
this.loc = loc;
this.size = roundTo4096(len);
mapAndSetOffset();
}

//Callers should synchronize to avoid calls in the middle of this, but
//it is undesirable to synchronize w/ all access methods.
public void remap(long nLen) throws Exception {
unmmap.invoke(null, addr, this.size);
this.size = roundTo4096(nLen);
mapAndSetOffset();
}

public int getInt(long pos) {
return unsafe.getInt(pos + addr);
}

public long getLong(long pos) {
return unsafe.getLong(pos + addr);
}

public void putInt(long pos, int val) {
unsafe.putInt(pos + addr, val);
}

public void putLong(long pos, long val) {
unsafe.putLong(pos + addr, val);
}

//May want to have offset & length within data as well, for both of these
public void getBytes(long pos, byte[] data) {
unsafe.copyMemory(null, pos + addr, data, BYTE_ARRAY_OFFSET, data.length);
}

public void putBytes(long pos, byte[] data) {
unsafe.copyMemory(data, BYTE_ARRAY_OFFSET, null, pos + addr, data.length);
}

@Override
public void close() throws IOException {
try {
unmmap.invoke(null, addr, this.size);
} catch (Exception e) {
throw new IOException("unmap error", e);
}
}

}
4 changes: 4 additions & 0 deletions common/src/main/java/lee/study/down/util/PathUtil.java
Original file line number Diff line number Diff line change
Expand Up @@ -35,5 +35,9 @@ public class PathUtil {
if (OsUtil.isWindows() && ROOT_PATH.indexOf("/") == 0) {
ROOT_PATH = ROOT_PATH.substring(1);
}
if (ROOT_PATH.lastIndexOf("/") != ROOT_PATH.length() - 1
&& ROOT_PATH.lastIndexOf("\\") != ROOT_PATH.length() - 1) {
ROOT_PATH += "/";
}
}
}
5 changes: 5 additions & 0 deletions core/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -18,5 +18,10 @@
<artifactId>proxyee-down-common</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.xerial.larray</groupId>
<artifactId>larray_2.12</artifactId>
<version>0.4.0</version>
</dependency>
</dependencies>
</project>
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,7 @@
import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.Map;
import lee.study.down.constant.HttpDownStatus;
Expand All @@ -40,7 +37,7 @@
@AllArgsConstructor
public abstract class AbstractHttpDownBootstrap {

private static final Logger LOGGER = LoggerFactory.getLogger(AbstractHttpDownBootstrap.class);
protected static final Logger LOGGER = LoggerFactory.getLogger(AbstractHttpDownBootstrap.class);

protected static final String ATTR_CHANNEL = "channel";
protected static final String ATTR_FILE_CHANNELS = "fileChannels";
Expand Down Expand Up @@ -86,7 +83,7 @@ public void startDown() throws Exception {
}
}

public void startChunkDown(ChunkInfo chunkInfo, int updateStatus) throws Exception {
protected void startChunkDown(ChunkInfo chunkInfo, int updateStatus) throws Exception {
HttpRequestInfo requestInfo = (HttpRequestInfo) httpDownInfo.getRequest();
RequestProto requestProto = requestInfo.requestProto();
LOGGER.debug("开始下载:" + chunkInfo);
Expand Down Expand Up @@ -184,12 +181,12 @@ public void pauseDown() throws Exception {
taskInfo.setStatus(HttpDownStatus.PAUSE);
for (ChunkInfo chunkInfo : taskInfo.getChunkInfoList()) {
synchronized (chunkInfo) {
close(chunkInfo);
if (chunkInfo.getStatus() != HttpDownStatus.DONE) {
chunkInfo.setStatus(HttpDownStatus.PAUSE);
}
}
}
close();
}
if (callback != null) {
callback.onPause(httpDownInfo);
Expand Down Expand Up @@ -227,12 +224,15 @@ public void continueDown()
}
}

public abstract boolean continueDownHandle() throws Exception;
protected abstract boolean continueDownHandle() throws Exception;

public abstract void merge() throws Exception;

public void close(ChunkInfo chunkInfo) {
try {
if (!attr.containsKey(chunkInfo.getIndex())) {
return;
}
Channel channel = getChannel(chunkInfo);
Closeable[] fileChannels = getFileWriter(chunkInfo);
LOGGER.debug(
Expand Down Expand Up @@ -291,21 +291,22 @@ protected Object getAttr(ChunkInfo chunkInfo, String key) {
}
}

public void setChannel(ChunkInfo chunkInfo, Channel channel) {
protected void setChannel(ChunkInfo chunkInfo, Channel channel) {
setAttr(chunkInfo, ATTR_CHANNEL, channel);
}

public Channel getChannel(ChunkInfo chunkInfo) {
return (Channel) getAttr(chunkInfo, ATTR_CHANNEL);
}

public abstract void initBoot() throws Exception;
protected abstract void initBoot() throws Exception;

public abstract Closeable[] initFileWriter(ChunkInfo chunkInfo) throws Exception;

public abstract boolean doFileWriter(ChunkInfo chunkInfo, ByteBuffer buffer) throws IOException;
public abstract int doFileWriter(ChunkInfo chunkInfo, ByteBuffer buffer)
throws IOException;

public Closeable[] getFileWriter(ChunkInfo chunkInfo) {
protected Closeable[] getFileWriter(ChunkInfo chunkInfo) {
return (Closeable[]) getAttr(chunkInfo, ATTR_FILE_CHANNELS);
}
}
59 changes: 27 additions & 32 deletions core/src/main/java/lee/study/down/boot/X64HttpDownBootstrap.java
Original file line number Diff line number Diff line change
Expand Up @@ -6,17 +6,17 @@
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.FileChannel.MapMode;
import lee.study.down.dispatch.HttpDownCallback;
import lee.study.down.io.LargeMappedByteBuffer;
import lee.study.down.io.Mmap;
import lee.study.down.model.ChunkInfo;
import lee.study.down.model.HttpDownInfo;
import lee.study.down.model.TaskInfo;
import lee.study.down.util.FileUtil;

public class X64HttpDownBootstrap extends AbstractHttpDownBootstrap {

private Mmap mmap;

public X64HttpDownBootstrap(HttpDownInfo httpDownInfo,
int retryCount,
SslContext clientSslContext,
Expand Down Expand Up @@ -54,40 +54,35 @@ public void initBoot() throws IOException {

@Override
public Closeable[] initFileWriter(ChunkInfo chunkInfo) throws Exception {
Closeable[] fileChannels;
FileChannel fileChannel = new RandomAccessFile(
getHttpDownInfo().getTaskInfo().buildTaskFilePath(), "rw")
.getChannel();
if (getHttpDownInfo().getTaskInfo().getConnections() > 1) {
LargeMappedByteBuffer mappedBuffer = new LargeMappedByteBuffer(fileChannel,
MapMode.READ_WRITE, chunkInfo.getNowStartPosition(),
chunkInfo.getEndPosition() - chunkInfo.getNowStartPosition() + 1);
fileChannels = new Closeable[]{fileChannel, mappedBuffer};
} else {
fileChannels = new Closeable[]{fileChannel};
if (mmap == null) {
mmap = new Mmap(getHttpDownInfo().getTaskInfo().buildTaskFilePath(),
getHttpDownInfo().getTaskInfo().getTotalSize());
}
return null;
}

@Override
public int doFileWriter(ChunkInfo chunkInfo, ByteBuffer buffer) throws IOException {
int ret = -1;
if (mmap != null) {
ret = buffer.remaining();
byte[] bts = new byte[buffer.remaining()];
buffer.get(bts);
mmap.putBytes(chunkInfo.getNowStartPosition() + chunkInfo.getDownSize(), bts);
}
setAttr(chunkInfo, ATTR_FILE_CHANNELS, fileChannels);
return fileChannels;
return ret;
}

@Override
public boolean doFileWriter(ChunkInfo chunkInfo, ByteBuffer buffer) throws IOException {
Closeable[] fileChannels = getFileWriter(chunkInfo);
if (fileChannels != null) {
if (fileChannels.length > 1) {
LargeMappedByteBuffer mappedBuffer = (LargeMappedByteBuffer) getFileWriter(chunkInfo)[1];
if (mappedBuffer != null) {
mappedBuffer.put(buffer);
return true;
}
} else {
FileChannel fileChannel = (FileChannel) getFileWriter(chunkInfo)[0];
if (fileChannel != null) {
fileChannel.write(buffer);
return true;
}
public void close() {
super.close();
if (mmap != null) {
try {
mmap.close();
mmap = null;
} catch (IOException e) {
LOGGER.error("mmap close error", e);
}
}
return false;
}
}
Loading

0 comments on commit 1b84586

Please sign in to comment.