From 0bebf8ba8a265c721db135de270c5586c42ac810 Mon Sep 17 00:00:00 2001 From: dilap54 Date: Thu, 21 Dec 2023 22:56:24 +0300 Subject: [PATCH] proxy --- cmd/cli/digi.go | 3 ++- cmd/cli/plati.go | 5 ++-- plati/categories.go | 4 +-- plati/client.go | 16 ++++++++--- plati/goodscategory.go | 28 +++++++++++++++++++- proxies.list | 6 +++++ proxies/proxies.go | 60 ++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 111 insertions(+), 11 deletions(-) create mode 100644 proxies.list create mode 100644 proxies/proxies.go diff --git a/cmd/cli/digi.go b/cmd/cli/digi.go index 35f654f..abcd382 100644 --- a/cmd/cli/digi.go +++ b/cmd/cli/digi.go @@ -8,6 +8,7 @@ import ( "gitea.home.4it.me/dilap54/platiparser/gorm" "gitea.home.4it.me/dilap54/platiparser/plati" + "gitea.home.4it.me/dilap54/platiparser/proxies" "github.com/urfave/cli/v2" "gorm.io/gorm/clause" ) @@ -21,7 +22,7 @@ var digiCommand = &cli.Command{ Action: func(c *cli.Context) error { ctx := context.Background() - platiCli := plati.New() + platiCli := plati.New(proxies.Default()) db := gorm.GetDB() diff --git a/cmd/cli/plati.go b/cmd/cli/plati.go index 07c5e5c..0b9f256 100644 --- a/cmd/cli/plati.go +++ b/cmd/cli/plati.go @@ -11,6 +11,7 @@ import ( "gitea.home.4it.me/dilap54/platiparser/gorm" "gitea.home.4it.me/dilap54/platiparser/plati" + "gitea.home.4it.me/dilap54/platiparser/proxies" uuid "github.com/satori/go.uuid" "github.com/urfave/cli/v2" "golang.org/x/sync/errgroup" @@ -29,7 +30,7 @@ var platiCommand = &cli.Command{ ctx := context.Background() db := gorm.GetDB() - platiCli := plati.New() + platiCli := plati.New(proxies.Default()) // printNames(categories) @@ -62,8 +63,6 @@ var platiCommand = &cli.Command{ } return wg.Wait() - - return nil }, } diff --git a/plati/categories.go b/plati/categories.go index bc624e1..825a442 100644 --- a/plati/categories.go +++ b/plati/categories.go @@ -76,7 +76,7 @@ func (c *Client) GetCategories(ctx context.Context) (*CategoriesResponse, error) } req.Header.Add("Accept", "application/json") - resp, err := c.httpCli.Do(req) + resp, err := c.httpCli().Do(req) if err != nil { return nil, fmt.Errorf("http do: %w", err) } @@ -108,7 +108,7 @@ func (c *Client) GetSubCategories(ctx context.Context, categoryID int) (*SubCate } req.Header.Add("Accept", "application/json") - resp, err := c.httpCli.Do(req) + resp, err := c.httpCli().Do(req) if err != nil { return nil, fmt.Errorf("http do: %w", err) } diff --git a/plati/client.go b/plati/client.go index 5a037d4..aab8dfe 100644 --- a/plati/client.go +++ b/plati/client.go @@ -1,17 +1,25 @@ package plati -import "net/http" +import ( + "math/rand" + "net/http" +) type Client struct { - httpCli *http.Client + httpClis []*http.Client token string sellerID int } -func New() *Client { +func New(clients []*http.Client) *Client { return &Client{ - httpCli: &http.Client{}, + httpClis: clients, token: "7C731D89FED84B479B89F24F81BB8AF2", sellerID: 1209592, } } + +func (c *Client) httpCli() *http.Client { + i := rand.Intn(len(c.httpClis)) + return c.httpClis[i] +} diff --git a/plati/goodscategory.go b/plati/goodscategory.go index ee5768f..b2d5c07 100644 --- a/plati/goodscategory.go +++ b/plati/goodscategory.go @@ -24,7 +24,7 @@ func (c *Client) GetBlockGoodsCategory(ctx context.Context, idC int, idR int, so dump, err := httputil.DumpRequestOut(req, false) log.Printf("%s\n", string(dump)) - resp, err := c.httpCli.Do(req) + resp, err := c.doWithRetry(req, 200) if err != nil { return nil, fmt.Errorf("http do: %w", err) } @@ -46,6 +46,32 @@ func (c *Client) GetBlockGoodsCategory(ctx context.Context, idC int, idR int, so return goods, nil } +func (c *Client) doWithRetry(req *http.Request, expectedCode int) (*http.Response, error) { + attempt := 0 + maxAttempts := 5 + for { + resp, err := c.httpCli().Do(req) + if err != nil { + if attempt < maxAttempts-1 { + continue + } else { + return resp, err + } + } + if resp.StatusCode != expectedCode { + if attempt < maxAttempts-1 { + resp.Body.Close() + continue + } else { + return resp, err + } + + } + + return resp, nil + } +} + type Good struct { Name string GoodLink string diff --git a/proxies.list b/proxies.list new file mode 100644 index 0000000..4fe70f7 --- /dev/null +++ b/proxies.list @@ -0,0 +1,6 @@ +socks5://aAymJo:Z2DJNN@193.124.191.198:9174 +socks5://aAymJo:Z2DJNN@193.124.191.216:9114 +socks5://aAymJo:Z2DJNN@194.67.202.63:9267 +socks5://aAymJo:Z2DJNN@194.67.217.9:9369 +socks5://aAymJo:Z2DJNN@193.124.190.68:9793 +socks5://atc622:Cheg1Y@46.8.248.8:9311 diff --git a/proxies/proxies.go b/proxies/proxies.go new file mode 100644 index 0000000..371cb94 --- /dev/null +++ b/proxies/proxies.go @@ -0,0 +1,60 @@ +package proxies + +import ( + "bufio" + "fmt" + "io" + "log" + "net/http" + "net/url" + "os" +) + +//https://www.reddit.com/r/golang/comments/ezg1ka/how_can_i_specify_a_proxy_server_when_using/ + +func Default() []*http.Client { + clis, err := ParseProxiesFromFile("./proxies.list") + if err != nil { + log.Fatal(err) + } + return clis +} + +func ParseProxiesFromFile(fileName string) ([]*http.Client, error) { + f, err := os.OpenFile(fileName, os.O_RDONLY, 0400) + if err != nil { + return nil, err + } + defer f.Close() + + return ParseProxies(f) +} + +func ParseProxies(r io.Reader) ([]*http.Client, error) { + clients := make([]*http.Client, 0) + + scanner := bufio.NewScanner(r) + + for scanner.Scan() { + proxyURL, err := url.Parse(scanner.Text()) + if err != nil { + return nil, fmt.Errorf("url parse %s: %w", scanner.Text(), err) + } + + transport := &http.Transport{ + Proxy: http.ProxyURL(proxyURL), + } + + client := &http.Client{Transport: transport} + + clients = append(clients, client) + + log.Printf("using %s as proxy", scanner.Text()) + } + + if len(clients) == 0 { + clients = append(clients, &http.Client{}) + } + + return clients, nil +}