From c472f67194c359d172d3abe5be1fc1835090b9ed Mon Sep 17 00:00:00 2001 From: Alex Fan Date: Fri, 10 Apr 2015 23:26:28 -0700 Subject: [PATCH] multi-thread downloading --- .../{Downloader.java => DownloadThread.java} | 15 +++++++-- slave/Slave.java | 33 +++++++++++++++++-- 2 files changed, 44 insertions(+), 4 deletions(-) rename slave/{Downloader.java => DownloadThread.java} (84%) diff --git a/slave/Downloader.java b/slave/DownloadThread.java similarity index 84% rename from slave/Downloader.java rename to slave/DownloadThread.java index 4fbc353..ca380ea 100644 --- a/slave/Downloader.java +++ b/slave/DownloadThread.java @@ -8,11 +8,22 @@ import javax.net.ssl.HttpsURLConnection; -public class Downloader { +public class DownloadThread extends Thread{ private static final String USER_AGENT = "Mozilla/5.0"; private static final String parentURL = "http://www.zhihu.com/question/"; + private int num; + + public DownloadThread(int num){ + this.num = num; + } + + @Override + public void run(){ + downloadByGet(num); + } + // HTTP GET request - public static void downloadByGet(int id){ + private static void downloadByGet(int id){ PrintWriter out = null; try { diff --git a/slave/Slave.java b/slave/Slave.java index de10bd3..ddcd0f8 100644 --- a/slave/Slave.java +++ b/slave/Slave.java @@ -1,7 +1,36 @@ +import java.util.*; +import java.awt.event.*; + public class Slave { public static void main (String[] args) { - Downloader.downloadByGet(20727479); - HTMLParser.getQuestionsIDs(20727479); + //DownloadService service = new DownloadService(); + //Thread t = new Thread(service); + //service.start(); + + TreeSet idset = new TreeSet(); + idset.add(22266015); + idset.add(19566985); + idset.add(21826476); + idset.add(19561631); + idset.add(19566980); + idset.add(19566985); + idset.add(19801697); + idset.add(19810462); + idset.add(19902415); + idset.add(20014415); + idset.add(20120168); + idset.add(20187542); + idset.add(20200749); + idset.add(20261859); + idset.add(20376438); + idset.add(20727479); + + + while(!idset.isEmpty()){ + Thread t = new DownloadThread(idset.pollFirst()); + t.start(); + } + System.err.println("Done"); }