Merge pull request robertdavidgraham#244 from mzpqnxow/feature.dedup.…

…20160728

[PATCH] FEATURE: Deduplicate ICMP like SYN/ACK and increase deduplication bucket size

src/main-dedup.c

@@ -13,7 +13,7 @@
 #include <stdlib.h>
 #include <string.h>
 
-#define DEDUP_ENTRIES 4096
+#define DEDUP_ENTRIES 65536 /* more aggressive deduplication */
 
 struct DedupEntry
 {

src/proto-icmp.c

@@ -5,6 +5,7 @@
 #include "output.h"
 #include "masscan-status.h"
 #include "templ-port.h"
+#include "main-dedup.h"
 
 
 /***************************************************************************
@@ -65,6 +66,13 @@ handle_icmp(struct Output *out, time_t timestamp,
     unsigned ip_them;
     unsigned cookie;
 
+    /* dedup ICMP echo replies as well as SYN/ACK replies */
+    static struct DedupTable *echo_reply_dedup = NULL;
+
+
+    if (!echo_reply_dedup)
+        echo_reply_dedup = dedup_create();
+
     ip_me = parsed->ip_dst[0]<<24 | parsed->ip_dst[1]<<16
             | parsed->ip_dst[2]<< 8 | parsed->ip_dst[3]<<0;
     ip_them = parsed->ip_src[0]<<24 | parsed->ip_src[1]<<16
@@ -81,6 +89,9 @@ handle_icmp(struct Output *out, time_t timestamp,
         if ((cookie & 0xFFFFFFFF) != seqno_me)
             return; /* not my response */
 
+        if (dedup_is_duplicate(echo_reply_dedup, ip_them, 0, ip_me, 0))
+            break;
+
         //if (syn_hash(ip_them, Templ_ICMP_echo) != seqno_me)
         //    return; /* not my response */