#!/bin/sh data_dir="data" usage() { cat <<- EOF Usage: ${0##*/} [options] Scrape reddit to block new nsfw domains Options: -o outfile, (default hosts) -p number of pages to scrape, (default 10) -d data directory, (default ./data) -l subreddit list to scrape, (default v2) v1 (1k subreddits) v2 (8k subreddits) bak (use the list in ${data_dir}/subs.bak.gz) EOF } note() { # usage note [color] [string1] [string2] ... : "${2:?}" color="${1:?}" shift case "$color" in "green") printf "\033[32;1m%b\033[0m" "$*" 1>&2 ;; "yellow") printf "\033[33;1m%b\033[0m" "$*" >&2 ;; "red") printf "\033[31;1m%b\033[0m" "$*" 1>&2 ;; *) printf "%b" "$*" 1>&2 ;; esac } dep_check() { # usage: cmd_check cmd_1 ... : "${1:?}" for dep; do if ! command -v "$dep" 1>/dev/null; then note "red" "err: $dep not found, please install it\n" exit 127 fi done unset dep } get_curl() { curl "$@" \ --silent \ --compressed \ --header 'User-Agent: Mozilla/5.0 (Windows NT 10.0; rv:112.0) Gecko/20100101 Firefox/112.0' } get_reddit() { # 'Cookie:_options={"pref_gated_sr_optin": true, "pref_quarantine_optin": true}; over18=1' get_curl "$1" \ --header 'Cookie: _options=%7B%22pref_gated_sr_optin%22%3A%20true%2C%20%22pref_quarantine_optin%22%3A%20true%7D; over18=1' \ --write-out "%{http_code}" } get_subs_v1() { note "white" "scrapping nsfw subreddit list\n" for page in \# $(seq 2 5); do get_reddit "https://redditlist.com/nsfw${page}.html" | grep -o 'reddit.com/r/[^"]*' | grep -v '^reddit.com/r/Redditlist/$' done | sort -u } get_subs_v2() { data= page=1 note "white" "scrapping nsfw subreddit list (v2)\n" while true; do data="$(get_curl "https://be.mysubs.cc/api/subs/?page=${page}")" for sub in $( echo "$data" | jq -r '.results[].name'); do printf "reddit.com/r/%s\n" "$sub" done if [ -n "$data" ]; then page=$((page + 1)) [ "$(echo "$data" | jq -r '.next')" = "null" ] && break; fi done } get_subs_bak() { zcat "${data_dir}/subs.bak.gz" } soutlinks() { # usage post_otlinks "reddit.com/r/subreddit" [pages] : "${1:?}" pages="${2:?}" page=0 note "white" "${1#*/} " while [ "$page" -lt "$pages" ] ; do data="$(get_reddit "https://old.${1}/?count=$((page * 25))")" && page=$((page + 1)) if [ -n "$data" ]; then : case "$data" in *"404") note "yellow" " banned subreddit" break ;; *"451") note "yellow" " banned in your country" break ;; *"403") note "red" " private subreddit" break ;; *"429") note "red" "..too many requests" page=$((page - 1)) continue ;; *"200") ;; *) note "red" "..something went wrong" page=$((page - 1)) continue ;; esac else note "red" "..empty" continue fi echo "$data" | grep -Eo 'class="title may-blank outbound" [^ ]* href="https?://[^"]*' | sed -E -e 's/.*href="https?:\/\///g' -e 's/\/.*//g' note "green" "..${page}" done note "white" "\n" } make_hosts() { # usage make_hosts [outfile] : "${1:?}" while read -r domain; do printf "0.0.0.0 %s\n" "$domain" done | tee "$1" } clean() { # usage clean [outfile] [domain_list] : "${1:?}" : "${2:?}" note "white" "removing duplicates and known good domains\n" curl https://raw.githubusercontent.com/Sinfonietta/hostfiles/master/pornography-hosts \ -o "${data_dir}/pornography-hosts" while read -r domain; do ! grep -i "$domain" "${data_dir}/pornography-hosts" "${data_dir}/whitelist" > /dev/null && echo "$domain" done < "$2" | make_hosts "$1" } main() { out="hosts" pages=10 list_fun="v2" temp_fl="$(mktemp "${TMPDIR:-/tmp}/redditorsbtfo.XXX")" dep_check curl jq grep sed while getopts "ho:p:l:d:" f; do case "$f" in o) out="$OPTARG" ;; p) pages="$OPTARG" ;; l) case "$OPTARG" in "v1") list_fun="v1" ;; "v2") list_fun="v2" ;; "bak") list_fun="bak" ;; *) usage exit 1 ;; esac ;; d) data_dir="$OPTARG" mkdir -p "$OPTARG" > /dev/null 2>&1 || exit 1 ;; h) usage exit ;; ?) usage exit 1 ;; esac done for sub in $(get_subs_${list_fun}); do soutlinks "$sub" "$pages" done | sort -u > "$temp_fl" clean "$out" "$temp_fl" rm "$temp_fl" } main "$@"