90 lines
3.0 KiB
C#
90 lines
3.0 KiB
C#
 
|
using System.Text.RegularExpressions;
|
||
|
|
|
||
|
|
namespace PrivaPub.Federation.Crawler
|
||
|
|
{
|
||
|
|
//RFC 9309: the most specific group naming the crawler (or "*"), the longest matching rule, Allow winning a tie.
|
||
|
|
//A 4xx means no rules at all; a 5xx or no answer means keep out.
|
||
|
|
public sealed class RobotsTxt
|
||
|
|
{
|
||
|
|
sealed record Rule(bool Allow, string Pattern, Regex Match);
|
||
|
|
|
||
|
|
readonly List<Rule> _rules;
|
||
|
|
readonly bool _everythingDisallowed;
|
||
|
|
|
||
|
|
RobotsTxt(List<Rule> rules, bool everythingDisallowed)
|
||
|
|
{
|
||
|
|
_rules = rules;
|
||
|
|
_everythingDisallowed = everythingDisallowed;
|
||
|
|
}
|
||
|
|
|
||
|
|
public static readonly RobotsTxt AllowAll = new(new List<Rule>(), false);
|
||
|
|
public static readonly RobotsTxt DisallowAll = new(new List<Rule>(), true);
|
||
|
|
|
||
|
|
public static RobotsTxt From(int status, string text, params string[] agents) => status switch
|
||
|
|
{
|
||
|
|
>= 200 and < 300 => Parse(text, agents),
|
||
|
|
>= 400 and < 500 => AllowAll,
|
||
|
|
_ => DisallowAll
|
||
|
|
};
|
||
|
|
|
||
|
|
public static RobotsTxt Parse(string text, params string[] agents)
|
||
|
|
{
|
||
|
|
var groups = new List<(List<string> Agents, List<Rule> Rules)>();
|
||
|
|
(List<string> Agents, List<Rule> Rules) current = default;
|
||
|
|
var collectingAgents = false;
|
||
|
|
foreach (var raw in (text ?? string.Empty).Split('\n'))
|
||
|
|
{
|
||
|
|
var line = raw.Split('#')[0].Trim();
|
||
|
|
var colon = line.IndexOf(':');
|
||
|
|
if (colon <= 0)
|
||
|
|
continue;
|
||
|
|
var field = line[..colon].Trim().ToLowerInvariant();
|
||
|
|
var value = line[(colon + 1)..].Trim();
|
||
|
|
if (field == "user-agent")
|
||
|
|
{
|
||
|
|
if (!collectingAgents)
|
||
|
|
{
|
||
|
|
current = (new List<string>(), new List<Rule>());
|
||
|
|
groups.Add(current);
|
||
|
|
collectingAgents = true;
|
||
|
|
}
|
||
|
|
current.Agents.Add(value.ToLowerInvariant());
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
if (field is not ("allow" or "disallow") || current.Agents == default)
|
||
|
|
continue;
|
||
|
|
collectingAgents = false;
|
||
|
|
if (value.Length == 0)
|
||
|
|
continue;
|
||
|
|
current.Rules.Add(new Rule(field == "allow", value, Pattern(value)));
|
||
|
|
}
|
||
|
|
|
||
|
|
foreach (var agent in agents.Select(a => a.ToLowerInvariant()))
|
||
|
|
{
|
||
|
|
var named = groups.Where(g => g.Agents.Contains(agent)).SelectMany(g => g.Rules).ToList();
|
||
|
|
if (groups.Any(g => g.Agents.Contains(agent)))
|
||
|
|
return new RobotsTxt(named, false);
|
||
|
|
}
|
||
|
|
return new RobotsTxt(groups.Where(g => g.Agents.Contains("*")).SelectMany(g => g.Rules).ToList(), false);
|
||
|
|
}
|
||
|
|
|
||
|
|
public bool Allows(string path)
|
||
|
|
{
|
||
|
|
if (_everythingDisallowed)
|
||
|
|
return false;
|
||
|
|
path = string.IsNullOrEmpty(path) ? "/" : path;
|
||
|
|
Rule best = default;
|
||
|
|
foreach (var rule in _rules.Where(r => r.Match.IsMatch(path)))
|
||
|
|
if (best == default || rule.Pattern.Length > best.Pattern.Length || rule.Pattern.Length == best.Pattern.Length && rule.Allow)
|
||
|
|
best = rule;
|
||
|
|
return best?.Allow != false;
|
||
|
|
}
|
||
|
|
|
||
|
|
static Regex Pattern(string value)
|
||
|
|
{
|
||
|
|
var anchored = value.EndsWith('$');
|
||
|
|
var body = Regex.Escape(anchored ? value[..^1] : value).Replace("\\*", ".*");
|
||
|
|
return new Regex("^" + body + (anchored ? "$" : string.Empty), RegexOptions.CultureInvariant, TimeSpan.FromMilliseconds(100));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|