crawler: ignore 405 responses for HEAD requests

It also skips header checks because they usually don't have headers we
need (for example, most omit the Content-Type header).

Fixes: https://todo.sr.ht/~sircmpwn/searchhut/41

crawler/index.go

@@ -175,6 +175,8 @@ func (c *Crawler) checkResponse(resp *http.Response, url *url.URL) bool {
 	switch resp.StatusCode {
 	case http.StatusOK:
 		// no-op
+	case http.StatusMethodNotAllowed:
+		return resp.Request.Method == "HEAD"
 	case http.StatusTooManyRequests:
 		retryAfter := resp.Header.Get("Retry-After")
 		if retryAfter == "" {