Colly Go package: how to check if the error is a Timeout error?

Everything is working except that I would like to retry only on Timeout errors, but I dont know how to compare with the specific Client.Timeout error that is generated.

The missing part is in comment here: if errors.Is(err, colly.Client.Timeout)...:

package main

import (
    "crypto/tls"
    "fmt"
    "github.com/gocolly/colly"
    "net/http"
    "os"
    "strings"
)

func main() {
    crawl()
}

func crawl() {
    httpMethod := "https"
    domains := []string{
        "www.myweb1.com",
        "www.myweb2.com",
    }
    for _, domain := range domains {
        // Instantiate default collector
        c := colly.NewCollector(
            colly.Async(true),
            colly.AllowedDomains(domain),
        )
        c.Limit(&colly.LimitRule{Parallelism: 100})
        /* default = 10s = 1000000000 nanoseconds: */
        c.SetRequestTimeout(10 * 1e9)
        c.WithTransport(&http.Transport{
            TLSClientConfig:&tls.Config{InsecureSkipVerify: true},
        })

        // On every a element which has href attribute call callback
        added := false
        c.OnHTML("a[href]", func(e *colly.HTMLElement) {
            url := e.Request.URL.String()
            link := e.Request.AbsoluteURL(e.Attr("href"))
            // Only those links are visited which are in AllowedDomains
            // create a new context to remember the referer (in case of error)
            for _, domain_ok := range domains {
                if (strings.Contains(link, domain_ok)) {
                    ctx := colly.NewContext()
                    ctx.Put("Referrer", url)
                    c.Request(http.MethodGet, link, nil, ctx, nil)
                    // Visit link found on page
                    c.Visit(link)
                    added = true
                    break
                }
            }
            if !added {
                fmt.Fprintf( os.Stdout, "Ignoring %s (%s)\n", link, url)
            }
        })

        // Before making a request print "Visiting ..."
        c.OnRequest(func(r *colly.Request) {
            fmt.Println("Visiting", r.URL.String())
        })
        c.OnError(func(resp *colly.Response, err error) {
            url := resp.Request.URL.String()
            fmt.Fprintf(
                os.Stdout, "ERR on URL: %s (from: %s), error: %s\n", url,
                resp.Request.Ctx.Get("Referrer"), err,
            )
            //if errors.Is(err, colly.Client.Timeout) {
            //    fmt.Fprintf(os.Stdout, "Retry: '%s'\n", url)
            //    r.Retry()
            //}
        })
        urlBase := fmt.Sprintf("%s://%s", httpMethod, domain)
        fmt.Println("Scraping: ", urlBase)
        c.Visit(urlBase)
        c.Wait()
    }
}

>Solution :

I guess you could use this snippet. Most likely the timeout error is threw because the deadline in context is exceeded. Could give a try.

import (
    "context"
    "os"

    "github.com/cockroachdb/errors"
)

func IsTimeoutError(err error) bool {
    if errors.Is(err, context.DeadlineExceeded) {
        return true
    }
    if errors.Is(errors.Cause(err), context.DeadlineExceeded) {
        return true
    }
    if os.IsTimeout(err) {
        return true
    }
    if os.IsTimeout(errors.Cause(err)) {
        return true
    }
    return false
}