crawler: ignore 405 responses for HEAD requests
It also skips header checks because they usually don't have headers we need (for example, most omit the Content-Type header). Fixes: https://todo.sr.ht/~sircmpwn/searchhut/41
This commit is contained in:
parent
8b11febce0
commit
b3a5803c0c
1 changed files with 2 additions and 0 deletions
|
@ -175,6 +175,8 @@ func (c *Crawler) checkResponse(resp *http.Response, url *url.URL) bool {
|
|||
switch resp.StatusCode {
|
||||
case http.StatusOK:
|
||||
// no-op
|
||||
case http.StatusMethodNotAllowed:
|
||||
return resp.Request.Method == "HEAD"
|
||||
case http.StatusTooManyRequests:
|
||||
retryAfter := resp.Header.Get("Retry-After")
|
||||
if retryAfter == "" {
|
||||
|
|
Loading…
Reference in a new issue