add parsing sumpays

cmd/cli/main.go

@@ -16,7 +16,7 @@ var commands = make([]*cli.Command, 0)
 
 func main() {
 	s := &cli.App{
-		Name:     "ricapi",
+		Name:     "platiparser",
 		Commands: commands,
 		Before: func(c *cli.Context) error {
 			godotenv.Load(".env")

cmd/cli/plati.go

@@ -6,11 +6,11 @@ import (
 	"fmt"
 	"log"
 	"os"
-	"strings"
 	"time"
 
 	"gitea.home.4it.me/dilap54/platiparser/gorm"
 	"gitea.home.4it.me/dilap54/platiparser/healthbeat"
+	"gitea.home.4it.me/dilap54/platiparser/internal/category"
 	"gitea.home.4it.me/dilap54/platiparser/plati"
 	"gitea.home.4it.me/dilap54/platiparser/proxies"
 	uuid "github.com/satori/go.uuid"
@@ -27,8 +27,8 @@ var platiCommand = &cli.Command{
 	Action: func(c *cli.Context) error {
 		timeStart := time.Now()
 
-		categories := openCategories("./categories.json").Content
-		categories = filterBySubstring("Gift", categories)
+		categories := category.OpenCategories("./categories.json").Content
+		categories = category.FilterBySubstring("Gift", categories)
 
 		ctx := context.Background()
 
@@ -101,22 +101,6 @@ func convertGoodsToGorm(cat *plati.Category, goods []*plati.Good) []*gorm.Good {
 	return out
 }
 
-func filterBySubstring(substring string, categories plati.Categories) plati.Categories {
-	out := make(plati.Categories, 0)
-	for _, c := range categories {
-		if len(c.Children) > 0 {
-			out = append(out, filterBySubstring(substring, c.Children)...)
-			continue
-		}
-
-		if strings.Contains(c.FlatName, substring) {
-			out = append(out, c)
-		}
-	}
-
-	return out
-}
-
 func printNames(categories plati.Categories) {
 	for _, c := range categories {
 		if len(c.Children) > 0 {
@@ -126,19 +110,3 @@ func printNames(categories plati.Categories) {
 		fmt.Printf("%s\n", c.FlatName)
 	}
 }
-
-func openCategories(fileName string) plati.CategoriesResponse {
-	f, err := os.OpenFile(fileName, os.O_RDONLY, 0400)
-	if err != nil {
-		panic(err)
-	}
-	defer f.Close()
-
-	out := plati.CategoriesResponse{}
-	if err := json.NewDecoder(f).Decode(&out); err != nil {
-		panic(err)
-	}
-	out.Content.FlatNames("")
-	out.Content.FixParentID(0)
-	return out
-}

cmd/cli/sumpay.go

@@ -0,0 +1,90 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"time"
+
+	"gitea.home.4it.me/dilap54/platiparser/gorm"
+	"gitea.home.4it.me/dilap54/platiparser/internal/category"
+	"gitea.home.4it.me/dilap54/platiparser/plati"
+	"gitea.home.4it.me/dilap54/platiparser/proxies"
+	uuid "github.com/satori/go.uuid"
+	"github.com/urfave/cli/v2"
+)
+
+func init() {
+	commands = append(commands, sumpayCommand)
+}
+
+var sumpayCommand = &cli.Command{
+	Name: "sumpay",
+	Action: func(c *cli.Context) error {
+		categories := category.OpenCategories("./categories.json").Content
+		categories = category.FilterBySubstring("Gift", categories)
+
+		cats := map[int]struct{}{}
+		for _, c := range categories {
+			cats[c.ParentID] = struct{}{}
+		}
+		catsArr := make([]int, 0, len(cats))
+		for cId := range cats {
+			catsArr = append(catsArr, cId)
+		}
+
+		ctx := context.Background()
+
+		db := gorm.GetDB()
+		platiCli := plati.New(proxies.Default())
+
+		// printNames(categories)
+
+		for i, categoryId := range catsArr {
+			for page := 1; page < 100; page++ {
+				log.Printf("fetching goods [%d/%d], page %d for %d\n", i, len(catsArr), page, categoryId)
+				goods, err := platiCli.GetGoods(ctx, categoryId, "ru-RU", page, 500, "RUR")
+				if err != nil {
+					return fmt.Errorf("GetGoods: %w", err)
+				}
+				log.Printf("inserting %d goods to DB", len(goods.Rows.Row))
+
+				if len(goods.Rows.Row) < 500 {
+					break
+				}
+
+				gormGoods := convertDigiGoodsToGorm(goods)
+				if err := db.Create(gormGoods).Error; err != nil {
+					return fmt.Errorf("db Create: %w", err)
+				}
+				log.Printf("inserted %d goods to DB", len(gormGoods))
+			}
+
+		}
+
+		return nil
+	},
+}
+
+func convertDigiGoodsToGorm(goods *plati.GoodsResponse) []*gorm.DigiGood {
+	out := make([]*gorm.DigiGood, 0, len(goods.Rows.Row))
+
+	for _, g := range goods.Rows.Row {
+		gormGood := gorm.DigiGood{
+			ID:           uuid.NewV4().String(),
+			IDGoods:      g.IDGoods,
+			Name:         g.NameGoods,
+			IDSection:    goods.IDSection,
+			SellerID:     g.IDSeller,
+			Sellerrating: g.Rating.Float64(),
+			Sold:         g.Statistics.CntSell,
+			Returned:     g.Statistics.CntReturn,
+			Price:        g.Price.Float64(),
+			Sumpay:       g.Summpay.Float64(),
+			CreatedAt:    time.Now(),
+		}
+		out = append(out, &gormGood)
+	}
+
+	return out
+}