21 lines
2.1 KiB
Bash
21 lines
2.1 KiB
Bash
 
|
# The opt-in crawler against the pasture's own peers. Needs PrivaPub started with the crawler on:
|
||
|
|
# PRIVAPUB_ENV="Statistics__Crawler__Enabled=true Statistics__Crawler__Seeds__0=mastodon.test" tools/pasture/run.sh up mastodon
|
||
|
|
# PrivaPub starts before its peers, so a first visit can find robots.txt answering 502 and rightly keep out for a week;
|
||
|
|
# the scenario then forgets that visit and restarts PrivaPub, which plans again.
|
||
|
|
echo "crawler"
|
||
|
|
privapub_root >/dev/null
|
||
|
|
podman exec -w /app pasture-privapub /app/PrivaPub admin promote "$ROOT_USER" >/dev/null 2>&1 || true
|
||
|
|
admin=$(curl -s -X POST $P/clientapi/user/login -H 'Content-Type: application/json' -d "{\"userName\":\"$ROOT_USER\",\"password\":\"$ROOT_PASS\"}" | j "print(d['token'])")
|
||
|
|
crawler() { curl -s -H "Authorization: Bearer $admin" "$P/clientapi/admin/statistics/crawler"; }
|
||
|
|
if [ "$(crawler | j "print(any(r['robotsDisallowed'] for r in d['recent']))")" = "True" ]; then
|
||
|
|
podman exec pasture-mongo mongosh --quiet PrivaPub --eval 'db.Job.deleteMany({Kind:{$in:[8,9]}}); db.RemoteInstance.updateMany({}, {$unset:{LastCrawledAt:1, RobotsDisallowed:1, CrawlError:1}})' >/dev/null
|
||
|
|
podman restart pasture-privapub >/dev/null
|
||
|
|
until_true 60 'curl -fs -o /dev/null $P/build.json'
|
||
|
|
fi
|
||
|
|
[ "$(crawler | j "print(d['enabled'])")" = "True" ] && ok "the crawler is on" || { ko "the crawler is off: start PrivaPub with PRIVAPUB_ENV=Statistics__Crawler__Enabled=true"; return 0; }
|
||
|
|
until_true 60 '[ "$(crawler | j "print(any(r[\"host\"]==\"mastodon.test\" and r[\"lastCrawledAt\"] for r in d[\"recent\"]))")" = "True" ]' \
|
||
|
|
&& ok "the crawler visited mastodon.test" || ko "the crawler never visited mastodon.test"
|
||
|
|
[ "$(crawler | j "print(next(r['peersCount'] is not None for r in d['recent'] if r['host']=='mastodon.test'))")" = "True" ] \
|
||
|
|
&& ok "it read mastodon.test's peers list" || ko "no peers list read from mastodon.test"
|
||
|
|
[ "$(curl -s -H "Authorization: Bearer $admin" "$P/clientapi/admin/statistics/hosts/mastodon.test?days=1" | j "print((d['instance'] or {}).get('software'))")" = "mastodon" ] \
|
||
|
|
&& ok "mastodon.test is described" || ko "mastodon.test is not described"
|