Gospider

Crawler in go

GO111MODULE=on go install github.com/jaeles-project/gospider@latest

#use httprobe against hosts from amass

gospider -S http_hosts.txt #run against list of hosts

#single target
gospider -s https://target.txt

#Requires anew (tool)
gospider -d 0 -s "https://site.com" -c 5 -t 100 -d 5 --blacklist jpg,jpeg,gif,css,tif,tiff,png,ttf,woff,woff2,ico,pdf,svg,txt | grep -Eo '(http|https)://[^/"]+' | anew