can backup

zfanCode · Apr 15, 2015 · 27d8f9f · 27d8f9f
1 parent c620dbb
commit 27d8f9f
Show file tree

Hide file tree

Showing 5 changed files with 127 additions and 18 deletions.
diff --git a/DownloadThread.java b/DownloadThread.java
@@ -26,10 +26,11 @@ public void run(){
 
 	// HTTP GET request
 	private void downloadByGet(){
-                PrintWriter out = null;
+        PrintWriter out = null;
+
+        try {
+            out =  new PrintWriter(new File("./download/"+num+".html"));
 
-                try {
-                    out =  new PrintWriter(new File("./download/"+num+".html"));
 
 		    String url = parentURL+num;
 
@@ -41,15 +42,13 @@ private void downloadByGet(){
 
 		    //add request header
 		    con.setRequestProperty("User-Agent", USER_AGENT);
-
 		    int responseCode = con.getResponseCode();
 		    System.out.println("\nSending 'GET' request to URL : " + url);
-
 		    BufferedReader in = new BufferedReader(
 		            new InputStreamReader(con.getInputStream()));
 		    String inputLine;
 		    StringBuffer response = new StringBuffer();
- 
+
 		    while ((inputLine = in.readLine()) != null) {
 		    	response.append(inputLine);
 		    }
@@ -58,13 +57,14 @@ private void downloadByGet(){
 		    //print result
 		    out.print(response.toString());
 
-                }catch(FileNotFoundException e) {
-                    System.err.println("File not found");
-                }catch(IOException e) {
-                    System.err.println(e.getMessage());
-                }finally{
-                    out.close();
-                }
+
+        }catch(FileNotFoundException e) {
+            System.err.println("File not found");
+        }catch(IOException e) {
+            System.err.println(e.getMessage());
+        }finally{
+            out.close();
+        }
 
 
                 TreeSet<Integer> idset = HTMLParser.getQuestionsIDs(num);

diff --git a/ExtractedIdAcceptService.java b/ExtractedIdAcceptService.java
@@ -10,11 +10,24 @@ public class ExtractedIdAcceptService implements Runnable,Protocol{
     private DataOutputStream toSlave;
     private static int count = 0;
 
+    private DataInputStream fromUniverseBk;
+    private DataInputStream fromDownloadQueueBk;
+    private DataOutputStream toUniverseBk;
+    private DataOutputStream toDownloadQueueBk;
+    private static final int BACKUP_FREQUENCY = 30;
 
-    public ExtractedIdAcceptService(TreeSet<Integer> universe, LinkedList<Integer> readyToDownload, Socket socket ){
+    public ExtractedIdAcceptService(TreeSet<Integer> universe, LinkedList<Integer> readyToDownload, Socket socket, DataInputStream fromUniverseBk,DataInputStream fromDownloadQueueBk, DataOutputStream toUniverseBk, DataOutputStream toDownloadQueueBk ){
         this.socket  = socket;
         this.universe = universe;
         this.readyToDownload = readyToDownload;
+
+        this.fromUniverseBk = fromUniverseBk;
+        this.fromDownloadQueueBk = fromDownloadQueueBk;
+        this.toUniverseBk = toUniverseBk;
+        this.toDownloadQueueBk =  toDownloadQueueBk;
+
+        count = getAmountDiff(universe, readyToDownload);
+
         try {
             fromSlave = new DataInputStream(socket.getInputStream());
             toSlave = new DataOutputStream(socket.getOutputStream());
@@ -45,6 +58,10 @@ public void run(){
                     toSlave.writeInt(readyToDownload.pollFirst());
                     toSlave.flush();
 
+                    if (count % BACKUP_FREQUENCY == 0) {
+                        System.out.println("backup ids");
+                        backup();
+                    }
                     count++;
                     System.err.println("Total downloaded: "+count);
                 }
@@ -59,4 +76,35 @@ public void run(){
         }
 
     }
+
+    private static int getAmountDiff(Set<Integer> universe, LinkedList<Integer> downloadList){
+        Set<Integer> result = new TreeSet<Integer>();
+        result.addAll(universe);
+        result.removeAll(downloadList);
+        return result.size();
+    }
+
+
+    private void backup(){
+
+        TreeSet<Integer> uni = new TreeSet<Integer>(universe);
+        LinkedList<Integer> download =new LinkedList<Integer>(readyToDownload);
+        try{
+            int tmp;
+            for (Integer  x:uni ) {
+                toUniverseBk.writeInt(x);
+            }
+            toUniverseBk.flush();
+
+
+            for (Integer  x:download ) {
+                toDownloadQueueBk.writeInt(x);
+            }
+            toDownloadQueueBk.flush();    
+        }catch(IOException e){
+            System.out.println("Cannot backup");
+            System.out.println(e.getMessage());
+        }
+
+    }
 }
diff --git a/Master.java b/Master.java
@@ -4,11 +4,24 @@
 
 public class Master implements Protocol{
     public static final int SEED = 22266015;
-    public static void main (String[] args) {
+    public static void main (String[] args)  throws FileNotFoundException{
         TreeSet<Integer> universe = new TreeSet<Integer>();
         LinkedList<Integer> readyToDownload = new LinkedList<Integer>();
         Socket socket = null;
         ServerSocket serversocket = null;
+        String UNIVERSE_BACKUP = "./data/IdUniverse.dat";
+        String READY_TO_DOWNLOAD_BACKUP = "./data/IdToDownload.dat";
+
+        DataInputStream fromUniverseBk = new DataInputStream(new FileInputStream(UNIVERSE_BACKUP));
+        DataInputStream fromDownloadQueueBk = new DataInputStream(new FileInputStream(READY_TO_DOWNLOAD_BACKUP));
+
+        init(universe, readyToDownload, fromUniverseBk,fromDownloadQueueBk);
+
+        DataOutputStream toUniverseBk = new DataOutputStream(new FileOutputStream(UNIVERSE_BACKUP));
+        DataOutputStream toDownloadQueueBk = new DataOutputStream(new FileOutputStream(READY_TO_DOWNLOAD_BACKUP));
+
+        backup(universe, readyToDownload, toUniverseBk,toDownloadQueueBk);
+
         try{
             serversocket = new ServerSocket(PORT);
         }catch(IOException e){
@@ -23,7 +36,7 @@ public static void main (String[] args) {
                     toSlave.writeInt(SEED);
                 }
                 toSlave.flush();
-                new Thread(new ExtractedIdAcceptService(universe,readyToDownload, socket)).start();
+                new Thread(new ExtractedIdAcceptService(universe,readyToDownload, socket, fromUniverseBk, fromDownloadQueueBk, toUniverseBk, toDownloadQueueBk)).start();
                 System.out.println("Connected");
 
             } catch(IOException e) {
@@ -33,4 +46,51 @@ public static void main (String[] args) {
         }
     }
 
+    private static void init(TreeSet<Integer> universe, LinkedList<Integer> readyToDownload,DataInputStream fromUniverseBk, DataInputStream fromDownloadQueueBk){
+        int tmp;
+        try {
+            while(true){
+                universe.add(fromUniverseBk.readInt());
+
+            }
+        }catch(Exception e){
+            System.err.println(e);
+        }
+        try {
+            while(true){
+                readyToDownload.add(fromDownloadQueueBk.readInt());
+            }
+        } catch(Exception e) {
+            System.err.println(e);
+        }
+
+        System.out.println("Finish initialization");
+
+
+    }
+
+    private static void backup(TreeSet<Integer> universe, LinkedList<Integer> readyToDownload, DataOutputStream toUniverseBk, DataOutputStream toDownloadQueueBk){
+
+        TreeSet<Integer> uni = new TreeSet<Integer>(universe);
+        LinkedList<Integer> download =new LinkedList<Integer>(readyToDownload);
+        try{
+            int tmp;
+            for (Integer  x:uni ) {
+                toUniverseBk.writeInt(x);
+            }
+            toUniverseBk.flush();
+
+
+            for (Integer  x:download ) {
+                toDownloadQueueBk.writeInt(x);
+            }
+            toDownloadQueueBk.flush();    
+        }catch(IOException e){
+            System.out.println("Cannot backup");
+            System.out.println(e.getMessage());
+        }
+
+    }
+
+
 }
diff --git a/Protocol.java b/Protocol.java
@@ -1,7 +1,8 @@
 public interface Protocol{
-    int PORT = 4127;
+    int PORT = 5252;
     int ID_REQUEST = 845689165;
     int MAX_THREADS = 8;
+    String IP = "localhost";
 
 
 }
diff --git a/Slave.java b/Slave.java
@@ -19,7 +19,7 @@ public Slave(Socket socket){
 
     public static void main (String[] args) {
         try {
-            Socket  socket = new Socket("192.168.0.18", PORT);
+            Socket  socket = new Socket(IP, PORT);
             new Thread(new Slave(socket)).start();
         } catch(Exception e) {
         }