Spiderfoot
SpiderFoot is a reconnaissance tool that automatically queries over 100 public data sources (OSINT) to gather intelligence on IP addresses, domain names, e-mail addresses, names and more.
https://www.spiderfoot.net/documentation/
https://github.com/smicallef/spiderfoot/
Installation with docker
sudo sh -c "echo -n 'lichnak:' >> /etc/nginx/.htpasswd"
sudo sh -c "openssl passwd -apr1 >> /etc/nginx/.htpasswd"
chmod 644 /etc/nginx/.htpasswd
cat > /etc/nginx/sites-available/spiderfoot.conf <<"EOF"
upstream spiderfoot {
server 127.0.0.1:5001 fail_timeout=0;
}
server {
server_name spiderfoot.mydomain.net;
access_log /var/log/nginx/vhost/spiderfoot.mydomain.net-access.log;
error_log /var/log/nginx/vhost/spiderfoot.mydomain.net-error.log;
auth_basic "Restricted";
auth_basic_user_file /etc/nginx/.htpasswd;
location / {
proxy_read_timeout 300;
proxy_connect_timeout 300;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $host;
proxy_set_header X-Forwarded-Server $host;
proxy_set_header Host $host;
proxy_set_header X-NginX-Proxy true;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_pass http://spiderfoot;
proxy_redirect off;
}
listen 443 ssl; # managed by Certbot
ssl_certificate /etc/letsencrypt/live/spiderfoot.mydomain.net/fullchain.pem; # managed by Certbot
ssl_certificate_key /etc/letsencrypt/live/spiderfoot.mydomain.net/privkey.pem; # managed by Certbot
include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
}
server {
if ($host = spiderfoot.mydomain.net) {
return 301 https://$host$request_uri;
} # managed by Certbot
server_name spiderfoot.mydomain.net;
listen 80;
return 404; # managed by Certbot
}
EOF
ln -s /etc/nginx/sites-enabled/spiderfoot.mydomain.net.conf /etc/nginx/sites-available/spiderfoot.mydomain.net.conf
nginx -t
systemctl restart nginx
certbot -d spiderfoot.mydomain.net --nginx
git clone https://github.com/smicallef/spiderfoot.git
cd spiderfoot/
docker-compose -f docker-compose.yml -f docker-compose-full.yml up
Upgrade from docker to systemd service
SPIDERFOOT_NAME="spiderfoot"
SPIDERFOOT_DATA="/var/lib/spiderfoot"
SPIDERFOOT_LOGS="/var/lib/spiderfoot/log"
SPIDERFOOT_CACHE="/var/lib/spiderfoot/cache"
SPIDERFOOT_HOME="/home/spiderfoot"
dnf config-manager --set-enabled powertools
dnf groupinstall "Development Tools" -y
dnf install -y openssl openssl-devel libxslt libxslt-devel tinyxml tinyxml-devel libxml2 libxml2-devel libjpeg libjpeg-devel zlib zlib-devel openjpeg2 openjpeg2-devel nmap tcpdump libunistring libunistring-devel libpcap libpcap-devel wget python39 python39-devel GeoIP-GeoLite-data libxslt-devel glibc-devel cmake gmp-devel gengetopt flex byacc json-c-devel swig curl git nbtscan testssl
groupadd $SPIDERFOOT_NAME
adduser -g $SPIDERFOOT_NAME -d $SPIDERFOOT_HOME -s /sbin/nologin $SPIDERFOOT_NAME
git clone https://github.com/smicallef/spiderfoot /tmp/$SPIDERFOOT_NAME
cp -ar /tmp/$SPIDERFOOT_NAME/* $SPIDERFOOT_HOME
chown -R spiderfoot:spiderfoot $SPIDERFOOT_HOME
mkdir -p $SPIDERFOOT_DATA || true
mkdir -p $SPIDERFOOT_LOGS || true
mkdir -p $SPIDERFOOT_CACHE || true
chown $SPIDERFOOT_NAME:$SPIDERFOOT_NAME $SPIDERFOOT_DATA
chown $SPIDERFOOT_NAME:$SPIDERFOOT_NAME $SPIDERFOOT_LOGS
chown $SPIDERFOOT_NAME:$SPIDERFOOT_NAME $SPIDERFOOT_CACHE
sudo -u spiderfoot bash
cd ~
pip3.9 install -U pip
pip3.9 install -r requirements.txt
SPIDERFOOT_CONTAINER_ID=$(docker ps -q)
DESTINATION_PATH="$HOME/backups"
docker cp "$SPIDERFOOT_CONTAINER_ID:$SPIDERFOOT_DATA" "DESTINATION_PATH"
docker stop $SPIDERFOOT_CONTAINER_ID
cp -ar $HOME/backups/* $SPIDERFOOT_DATA
cat >/etc/systemd/system/spiderfoot.service <<"EOF"
[Unit]
Description=SpiderFoot service.
[Service]
Type=simple
Restart=always
RestartSec=3
StartLimitBurst=5
User=spiderfoot
Group=spiderfoot
StandardOutput=syslog
StandardError=syslog
SyslogIdentifier=spiderfoot
ExecStart=python3.9 /home/spiderfoot/sf.py -l 127.0.0.1:8082
WorkingDirectory=/home/spiderfoot/
[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable --now spiderfoot
docker rm $(docker ps --all)
docker system prune --all
cd ~
curl -sL https://rpm.nodesource.com/setup_18.x -o nodesource_setup.sh
vi nodesource_setup.sh
sudo yum remove -y nodejs npm
sudo yum install gcc-c++ make
sudo dnf install -y nodejs
curl -sL https://dl.yarnpkg.com/rpm/yarn.repo | sudo tee /etc/yum.repos.d/yarn.repo
sudo yum install yarn
sudo dnf install nodejs -y
node -v