crawler: ignore 405 responses for HEAD requests
It also skips header checks because they usually don't have headers we need (for example, most omit the Content-Type header). Fixes: https://todo.sr.ht/~sircmpwn/searchhut/41
This commit is contained in:
parent
8b11febce0
commit
b3a5803c0c
1 changed files with 2 additions and 0 deletions
|
@ -175,6 +175,8 @@ func (c *Crawler) checkResponse(resp *http.Response, url *url.URL) bool {
|
||||||
switch resp.StatusCode {
|
switch resp.StatusCode {
|
||||||
case http.StatusOK:
|
case http.StatusOK:
|
||||||
// no-op
|
// no-op
|
||||||
|
case http.StatusMethodNotAllowed:
|
||||||
|
return resp.Request.Method == "HEAD"
|
||||||
case http.StatusTooManyRequests:
|
case http.StatusTooManyRequests:
|
||||||
retryAfter := resp.Header.Get("Retry-After")
|
retryAfter := resp.Header.Get("Retry-After")
|
||||||
if retryAfter == "" {
|
if retryAfter == "" {
|
||||||
|
|
Loading…
Reference in a new issue