removed dnsrecon, theharvester is back

install.sh

@@ -78,8 +78,7 @@ declare -A repos
 repos["uDork"]="m3n0sd0n4ld/uDork"
 repos["pwndb"]="davidtavarez/pwndb"
 repos["dnsvalidator"]="vortexau/dnsvalidator"
-repos["dnsrecon"]="darkoperator/dnsrecon"
-#repos["theHarvester"]="laramies/theHarvester"
+repos["theHarvester"]="laramies/theHarvester"
 repos["brutespray"]="x90skysn3k/brutespray"
 repos["wafw00f"]="EnableSecurity/wafw00f"
 repos["gf"]="tomnomnom/gf"
@@ -289,7 +288,7 @@ for repo in "${!repos[@]}"; do
         eval $SUDO pip3 install . $DEBUG_STD
     fi
     if [ -s "requirements.txt" ]; then
-        eval $SUDO pip3 install -r requirements.txt $DEBUG_STD
+        #eval $SUDO pip3 install -r requirements.txt $DEBUG_STD
         eval $SUDO python3 setup.py install --record files.txt $DEBUG_STD
         [ -s "files.txt" ] && eval xargs rm -rf < files.txt $DEBUG_STD
         eval $SUDO pip3 install . $DEBUG_STD
@@ -422,6 +421,6 @@ eval strip -s $HOME/go/bin/* $DEBUG_STD
 
 eval $SUDO cp $HOME/go/bin/* /usr/local/bin/ $DEBUG_STD
 
-printf "${yellow} Remember set your api keys:\n - amass (~/.config/amass/config.ini)\n - subfinder (~/.config/subfinder/config.yaml)\n - GitHub (~/Tools/.github_tokens)\n - SHODAN (SHODAN_API_KEY in reconftw.cfg or env var)\n - SSRF Server (COLLAB_SERVER in reconftw.cfg or env var) \n - Blind XSS Server (XSS_SERVER in reconftw.cfg or env var) \n - notify (~/.config/notify/provider-config.yaml) \n - theHarvester (~/Tools/theHarvester/api-keys.yml)\n - H8mail (~/Tools/h8mail_config.ini)\n - uDork FB cookie (UDORK_COOKIE in reconftw.cfg or env var)\n - WHOISXML API (WHOISXML_API in reconftw.cfg or env var)\n\n\n${reset}"
+printf "${yellow} Remember set your api keys:\n - amass (~/.config/amass/config.ini)\n - subfinder (~/.config/subfinder/config.yaml)\n - GitHub (~/Tools/.github_tokens)\n - SHODAN (SHODAN_API_KEY in reconftw.cfg or env var)\n - SSRF Server (COLLAB_SERVER in reconftw.cfg or env var) \n - Blind XSS Server (XSS_SERVER in reconftw.cfg or env var) \n - notify (~/.config/notify/provider-config.yaml) \n - theHarvester (~/Tools/theHarvester/api-keys.yaml or /etc/theHarvester/api-keys.yaml)\n - H8mail (~/Tools/h8mail_config.ini)\n - uDork FB cookie (UDORK_COOKIE in reconftw.cfg or env var)\n - WHOISXML API (WHOISXML_API in reconftw.cfg or env var)\n\n\n${reset}"
 printf "${bgreen} Finished!${reset}\n\n"
 printf "\n\n${bgreen}#######################################################################${reset}\n"

reconftw.sh

@@ -195,16 +195,15 @@ function emails(){
 		emailfinder -d $domain 2>>"$LOGFILE" | anew -q .tmp/emailfinder.txt
 		[ -s ".tmp/emailfinder.txt" ] && cat .tmp/emailfinder.txt | awk 'matched; /^-----------------$/ { matched = 1 }' | anew -q osint/emails.txt
 
-		#theHarvester commented until they resolves multiple bugs
-#		cd "$tools/theHarvester" || { echo "Failed to cd directory in ${FUNCNAME[0]} @ line ${LINENO}"; exit 1; }
-#		python3 theHarvester.py -d $domain -b "anubis, baidu, bing, binaryedge, bingapi, bufferoverun, censys, certspotter, crtsh, dnsdumpster, duckduckgo, github-code, google, hackertarget, hunter, intelx, linkedin, linkedin_links, netcraft, omnisint, otx, pentesttools, projectdiscovery, qwant, rapiddns, rocketreach, securityTrails, sublist3r, threatcrowd, threatminer, trello, twitter, urlscan, virustotal, yahoo, zoomeye" 2>>"$LOGFILE" > $dir/.tmp/harvester.txt
-#		cd "$dir" || { echo "Failed to cd to $dir in ${FUNCNAME[0]} @ line ${LINENO}"; exit 1; }
-#		if [ -s ".tmp/harvester.txt" ]; then
-#			cat .tmp/harvester.txt | awk '/Emails/,/Hosts/' | sed -e '1,2d' | head -n -2 | sed -e '/Searching /d' -e '/exception has occurred/d' -e '/found:/Q' | anew -q osint/emails.txt
-#			cat .tmp/harvester.txt | awk '/Users/,/IPs/' | sed -e '1,2d' | head -n -2 | sed -e '/Searching /d' -e '/exception has occurred/d' -e '/found:/Q' | anew -q osint/users.txt
-#			cat .tmp/harvester.txt | awk '/Links/,/Users/' | sed -e '1,2d' | head -n -2 | sed -e '/Searching /d' -e '/exception has occurred/d' -e '/found:/Q' | anew -q osint/linkedin.txt
-#		fi
-
+		theHarvester commented until they resolves multiple bugs
+		cd "$tools/theHarvester" || { echo "Failed to cd directory in ${FUNCNAME[0]} @ line ${LINENO}"; exit 1; }
+		python3 theHarvester.py -d $domain -b all -f $dir/.tmp/harvester.json >/dev/null
+		cd "$dir" || { echo "Failed to cd to $dir in ${FUNCNAME[0]} @ line ${LINENO}"; exit 1; }
+		if [ -s ".tmp/harvester.json" ]; then
+			cat .tmp/harvester.json | jq -r .emails[] 2>/dev/null | anew -q osint/emails.txt
+			cat .tmp/harvester.json | jq -r .linkedin_people[] 2>/dev/null | anew -q osint/employees.txt
+			cat .tmp/harvester.json | jq -r .linkedin_links[] 2>/dev/null | anew -q osint/linkedin.txt
+		fi
 		h8mail -t $domain -q domain --loose -c $tools/h8mail_config.ini -j .tmp/h8_results.json 2>>"$LOGFILE" &>/dev/null
 		[ -s ".tmp/h8_results.json" ] && cat .tmp/h8_results.json | jq -r '.targets[0] | .data[] | .[]' | cut -d '-' -f2 | anew -q osint/h8mail.txt
 
@@ -667,6 +666,8 @@ function subtakeover(){
 		cat .tmp/subs_no_resolved.txt .tmp/subdomains_dns.txt .tmp/scrap_subs.txt .tmp/analytics_subs_clean.txt .tmp/passive_recursive.txt 2>/dev/null | anew -q .tmp/subs_dns_tko.txt
 		cat .tmp/subs_dns_tko.txt 2>/dev/null | dnstake -c $DNSTAKE_THREADS -s 2>>"$LOGFILE" | anew -q .tmp/tko.txt
 
+		sed -i '/^$/d' .tmp/tko.txt
+
 		NUMOFLINES=$(cat .tmp/tko.txt 2>>"$LOGFILE" | anew webs/takeover.txt | wc -l)
 		if [ "$NUMOFLINES" -gt 0 ]; then
 			notification "${NUMOFLINES} new possible takeovers found" info
@@ -687,9 +688,9 @@ function subtakeover(){
 function zonetransfer(){
 	if { [ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ] || [ "$DIFF" = true ]; } && [ "$ZONETRANSFER" = true ] && ! [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9] ]]; then
 		start_func ${FUNCNAME[0]} "Zone transfer check"
-		python3 $tools/dnsrecon/dnsrecon.py -d $domain -a -j subdomains/zonetransfer.json 2>>"$LOGFILE" &>/dev/null
-		if [ -s "subdomains/zonetransfer.json" ]; then
-			if grep -q "\"zone_transfer\"\: \"success\"" subdomains/zonetransfer.json ; then notification "Zone transfer found on ${domain}!" info; fi
+		dig axfr $domain @8.8.8.8 > subdomains/zonetransfer.txt
+		if [ -s ".tmp/zone_transfer.txt" ]; then
+			if ! grep -q "Transfer failed" subdomains/zonetransfer.txt ; then notification "Zone transfer found on ${domain}!" info; fi
 		fi
 		end_func "Results are saved in $domain/subdomains/zonetransfer.txt" ${FUNCNAME[0]}
 	else
@@ -712,7 +713,7 @@ function s3buckets(){
 			[ -s "subdomains/subdomains.txt" ] && s3scanner scan -f subdomains/subdomains.txt | grep -iv "not_exist" | grep -iv "Warning:" | grep -iv "invalid_name" | anew -q .tmp/s3buckets.txt
 		else
 			axiom-scan webs/webs.txt -m s3scanner -o .tmp/s3buckets_tmp.txt $AXIOM_EXTRA_ARGS 2>>"$LOGFILE" &>/dev/null
-			[ -s ".tmp/s3buckets_tmp.txt" ] && cat .tmp/s3buckets_tmp.txt | grep -iv "not_exist" | grep -iv "Warning:" | anew -q .tmp/s3buckets.txt
+			[ -s ".tmp/s3buckets_tmp.txt" ] && cat .tmp/s3buckets_tmp.txt | grep -iv "not_exist" | grep -iv "Warning:" | anew -q .tmp/s3buckets.txt && sed -i '/^$/d' .tmp/s3buckets.txt
 		fi
 		# Cloudenum
 		keyword=${domain%%.*}

requirements.txt

@@ -1,42 +1,41 @@
-jsbeautifier
-mmh3
-ipwhois
-fake-useragent
-colorclass==2.2.0
-tqdm
-termcolor
-argparse
-future
-metafinder
-netaddr
-requests
-lxml
-PySocks==1.6.8
-h8mail
-argcomplete==1.10.0
-s3scanner
-shodan
-dnspython
-emailfinder
-requests_futures
-bbrf
-uro
-bs4
-urllib3
-aiodns
-aiofiles
-aiohttp
-aiomultiprocess
-aiosqlite
-beautifulsoup4
-censys
-certifi
-fastapi
-orjson
-pyppeteer
-PyYAML
-retrying
-setuptools
-slowapi
-starlette
-uvicorn
+aiodns                  # theHarvester
+aiofiles                # theHarvester
+aiohttp                 # theHarvester
+aiomultiprocess         # theHarvester
+aiosqlite               # theHarvester
+argcomplete             # brutespray
+argparse                # multiple
+bbrf                    # multiple
+beautifulsoup4          # multiple
+bs4                     # multiple
+censys                  # theHarvester
+certifi                 # theHarvester
+colorclass              # interlace
+dnspython               # ip2provider
+emailfinder             # Tool
+fake-useragent          # fav-up
+fastapi                 # theHarvester
+future                  # multiple
+h8mail                  # Tool
+ipwhois                 # fav-up
+jsbeautifier            # Linkfinder
+metafinder              # Tool
+mmh3                    # fav-up
+netaddr                 # dnsvalidator
+orjson                  # theHarvester
+pyppeteer               # theHarvester
+PySocks                 # pwndb
+PyYAML                  # theHarvester
+requests                # multiple
+requests_futures        # cloud_enums
+retrying                # theHarvester
+s3scanner               # Tool
+setuptools              # theHarvester
+shodan                  # Tool
+slowapi                 # theHarvester
+starlette               # theHarvester
+termcolor               # gitDorker
+tqdm                    # multiple
+urllib3                 # multiple
+uro                     # Tool
+uvicorn                 # theHarvester