proxy

cmd/cli/digi.go

@@ -8,6 +8,7 @@ import (
 
 	"gitea.home.4it.me/dilap54/platiparser/gorm"
 	"gitea.home.4it.me/dilap54/platiparser/plati"
+	"gitea.home.4it.me/dilap54/platiparser/proxies"
 	"github.com/urfave/cli/v2"
 	"gorm.io/gorm/clause"
 )
@@ -21,7 +22,7 @@ var digiCommand = &cli.Command{
 	Action: func(c *cli.Context) error {
 		ctx := context.Background()
 
-		platiCli := plati.New()
+		platiCli := plati.New(proxies.Default())
 
 		db := gorm.GetDB()
 

cmd/cli/plati.go

@@ -11,6 +11,7 @@ import (
 
 	"gitea.home.4it.me/dilap54/platiparser/gorm"
 	"gitea.home.4it.me/dilap54/platiparser/plati"
+	"gitea.home.4it.me/dilap54/platiparser/proxies"
 	uuid "github.com/satori/go.uuid"
 	"github.com/urfave/cli/v2"
 	"golang.org/x/sync/errgroup"
@@ -29,7 +30,7 @@ var platiCommand = &cli.Command{
 		ctx := context.Background()
 
 		db := gorm.GetDB()
-		platiCli := plati.New()
+		platiCli := plati.New(proxies.Default())
 
 		// printNames(categories)
 
@@ -62,8 +63,6 @@ var platiCommand = &cli.Command{
 		}
 
 		return wg.Wait()
-
-		return nil
 	},
 }
 

plati/categories.go

@@ -76,7 +76,7 @@ func (c *Client) GetCategories(ctx context.Context) (*CategoriesResponse, error)
 	}
 	req.Header.Add("Accept", "application/json")
 
-	resp, err := c.httpCli.Do(req)
+	resp, err := c.httpCli().Do(req)
 	if err != nil {
 		return nil, fmt.Errorf("http do: %w", err)
 	}
@@ -108,7 +108,7 @@ func (c *Client) GetSubCategories(ctx context.Context, categoryID int) (*SubCate
 	}
 	req.Header.Add("Accept", "application/json")
 
-	resp, err := c.httpCli.Do(req)
+	resp, err := c.httpCli().Do(req)
 	if err != nil {
 		return nil, fmt.Errorf("http do: %w", err)
 	}

plati/client.go

@@ -1,17 +1,25 @@
 package plati
 
-import "net/http"
+import (
+	"math/rand"
+	"net/http"
+)
 
 type Client struct {
-	httpCli  *http.Client
+	httpClis []*http.Client
 	token    string
 	sellerID int
 }
 
-func New() *Client {
+func New(clients []*http.Client) *Client {
 	return &Client{
-		httpCli:  &http.Client{},
+		httpClis: clients,
 		token:    "7C731D89FED84B479B89F24F81BB8AF2",
 		sellerID: 1209592,
 	}
 }
+
+func (c *Client) httpCli() *http.Client {
+	i := rand.Intn(len(c.httpClis))
+	return c.httpClis[i]
+}

plati/goodscategory.go

@@ -24,7 +24,7 @@ func (c *Client) GetBlockGoodsCategory(ctx context.Context, idC int, idR int, so
 	dump, err := httputil.DumpRequestOut(req, false)
 	log.Printf("%s\n", string(dump))
 
-	resp, err := c.httpCli.Do(req)
+	resp, err := c.doWithRetry(req, 200)
 	if err != nil {
 		return nil, fmt.Errorf("http do: %w", err)
 	}
@@ -46,6 +46,32 @@ func (c *Client) GetBlockGoodsCategory(ctx context.Context, idC int, idR int, so
 	return goods, nil
 }
 
+func (c *Client) doWithRetry(req *http.Request, expectedCode int) (*http.Response, error) {
+	attempt := 0
+	maxAttempts := 5
+	for {
+		resp, err := c.httpCli().Do(req)
+		if err != nil {
+			if attempt < maxAttempts-1 {
+				continue
+			} else {
+				return resp, err
+			}
+		}
+		if resp.StatusCode != expectedCode {
+			if attempt < maxAttempts-1 {
+				resp.Body.Close()
+				continue
+			} else {
+				return resp, err
+			}
+
+		}
+
+		return resp, nil
+	}
+}
+
 type Good struct {
 	Name         string
 	GoodLink     string

proxies.list

@@ -0,0 +1,6 @@
+socks5://aAymJo:Z2DJNN@193.124.191.198:9174
+socks5://aAymJo:Z2DJNN@193.124.191.216:9114
+socks5://aAymJo:Z2DJNN@194.67.202.63:9267
+socks5://aAymJo:Z2DJNN@194.67.217.9:9369
+socks5://aAymJo:Z2DJNN@193.124.190.68:9793
+socks5://atc622:Cheg1Y@46.8.248.8:9311

proxies/proxies.go

@@ -0,0 +1,60 @@
+package proxies
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"net/url"
+	"os"
+)
+
+//https://www.reddit.com/r/golang/comments/ezg1ka/how_can_i_specify_a_proxy_server_when_using/
+
+func Default() []*http.Client {
+	clis, err := ParseProxiesFromFile("./proxies.list")
+	if err != nil {
+		log.Fatal(err)
+	}
+	return clis
+}
+
+func ParseProxiesFromFile(fileName string) ([]*http.Client, error) {
+	f, err := os.OpenFile(fileName, os.O_RDONLY, 0400)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	return ParseProxies(f)
+}
+
+func ParseProxies(r io.Reader) ([]*http.Client, error) {
+	clients := make([]*http.Client, 0)
+
+	scanner := bufio.NewScanner(r)
+
+	for scanner.Scan() {
+		proxyURL, err := url.Parse(scanner.Text())
+		if err != nil {
+			return nil, fmt.Errorf("url parse %s: %w", scanner.Text(), err)
+		}
+
+		transport := &http.Transport{
+			Proxy: http.ProxyURL(proxyURL),
+		}
+
+		client := &http.Client{Transport: transport}
+
+		clients = append(clients, client)
+
+		log.Printf("using %s as proxy", scanner.Text())
+	}
+
+	if len(clients) == 0 {
+		clients = append(clients, &http.Client{})
+	}
+
+	return clients, nil
+}