add content-length filter

FreedomZZQ · Apr 28, 2016 · d9359ba · d9359ba
1 parent b1ec51a
commit d9359ba
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 0 deletions.
diff --git a/src/crawler/Crawler.java b/src/crawler/Crawler.java
@@ -73,6 +73,13 @@ public void processURL(String strUrl) {
 				log(TYPE_CONNECTING, url.toString(), TAG_ERROR);
 				return;
 			}
+
+			//过滤小于 CONTENT_LENGTH 的文件链接
+			if((connection.getContentLength() < CONTENT_LENGTH)){
+				log(TYPE_CONNECTING, url.toString(), TAG_ERROR);
+				return;
+			}
+
 			log(TYPE_CONNECTING, url.toString(), TAG_SUCCESS);
 
 			// read the URL

diff --git a/src/utils/Constants.java b/src/utils/Constants.java
@@ -14,4 +14,5 @@ public abstract class Constants {
     public static final String TAG_ERROR = "Error";
     public static final String LOG_FILE_NAME = "IR201330551365LOG.txt";
     public static final String CONTENT_TYPE = "text/";
+    public static final int CONTENT_LENGTH = 8000;
 }