package plati import ( "context" "fmt" "io" "log" "math/rand" "net/http" "net/http/httputil" "strconv" "strings" "github.com/PuerkitoBio/goquery" ) func (c *Client) GetBlockGoodsCategory(ctx context.Context, idC int, idR int, sort string, page int, rows int, curr string, lang string) ([]*Good, error) { u := fmt.Sprintf("https://plati.market/asp/block_goods_category.asp?preorders=0&id_cb=0&id_c=%d&id_r=%d&sort=%s&page=%d&rows=%d&curr=%s&pp_only=false&lang=ru-RU&rnd=%f", idC, idR, sort, page, rows, curr, rand.Float32()) req, err := http.NewRequestWithContext(ctx, "GET", u, http.NoBody) if err != nil { return nil, fmt.Errorf("new request: %w", err) } dump, err := httputil.DumpRequestOut(req, false) log.Printf("%s\n", string(dump)) resp, err := c.doWithRetry(req, 200) if err != nil { return nil, fmt.Errorf("http do: %w", err) } // dump, err = httputil.DumpResponse(resp, true) // log.Printf("%d %s", resp.StatusCode, string(dump)) defer resp.Body.Close() if resp.StatusCode != 200 { return nil, fmt.Errorf("status code: %d", resp.StatusCode) } goods, err := parseGoodsCategory(resp.Body) if err != nil { return nil, fmt.Errorf("parse goods category: %w", err) } return goods, nil } func (c *Client) doWithRetry(req *http.Request, expectedCode int) (*http.Response, error) { attempt := 0 maxAttempts := 5 for { resp, err := c.httpCli().Do(req) if err != nil { if attempt < maxAttempts-1 { continue } else { return resp, err } } if resp.StatusCode != expectedCode { if attempt < maxAttempts-1 { resp.Body.Close() continue } else { return resp, err } } return resp, nil } } type Good struct { Name string GoodLink string Seller string SellerLink string SellerRating int Sold int Price float64 } func parseGoodsCategory(r io.Reader) ([]*Good, error) { doc, err := goquery.NewDocumentFromReader(r) if err != nil { return nil, fmt.Errorf("goquery new document: %w", err) } goods := make([]*Good, 0) doc.Find("table.goods-table-category>tbody>tr").Each(func(i int, tr *goquery.Selection) { good := Good{} goods = append(goods, &good) tr.Find("td").Each(func(i int, td *goquery.Selection) { if td.HasClass("product-title") { titleBlock := td.Find("a").First() good.Name = titleBlock.Text() good.GoodLink, _ = titleBlock.Attr("href") } if td.HasClass("product-merchant") { titleBlock := td.Find("a").First() good.Seller = titleBlock.Text() good.SellerLink, _ = titleBlock.Attr("href") spanBlock := td.Find("span").First() sellerRating, err := strconv.ParseInt(spanBlock.Text(), 10, 64) if err != nil { log.Printf("cannot parse seller rating %s", spanBlock.Text()) } good.SellerRating = int(sellerRating) } if td.HasClass("product-sold") { sold, err := strconv.ParseInt(td.Text(), 10, 64) if err != nil { log.Printf("cannot parse product sold %s", td.Text()) } good.Sold = int(sold) } if td.HasClass("product-price") { productPriceText := strings.Fields(td.Text())[0] productPrice, err := strconv.ParseFloat(productPriceText, 64) if err != nil { log.Printf("cannot parse product price %s: %v", td.Text(), err) } good.Price = productPrice } }) }) return goods, nil }