dilap54/platiparser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

postgresql

Browse Source
This commit is contained in:
dilap54
2023-12-20 23:09:12 +03:00
parent 872217d845
commit 89046d130c
14 changed files with 336 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.env Normal file
View File

@ -0,0 +1 @@
POSTGRESQL_DSN="host=localhost port=5432 user=postgres password=123321Aa dbname=platiparser sslmode=disable"

25
cmd/cli/goose.go Normal file
View File

@ -0,0 +1,25 @@
package main
import (
"gitea.home.4it.me/dilap54/platiparser/gorm"
"github.com/pressly/goose"
"github.com/urfave/cli/v2"
)
func init() {
commands = append(commands, gooseCommand)
}
var gooseCommand = &cli.Command{
Name: "goose",
Action: func(c *cli.Context) error {
db, err := gorm.GetDB().DB()
if err != nil {
return err
}
if err := goose.Run(c.Args().First(), db, "./migrations", c.Args().Tail()...); err != nil {
return err
}
return nil
},
}

30
cmd/cli/main.go Normal file
View File

@ -0,0 +1,30 @@
package main
import (
"log"
"os"
"github.com/joho/godotenv"
"github.com/urfave/cli/v2"
)
func init() {
godotenv.Load(".env")
}
var commands = make([]*cli.Command, 0)
func main() {
s := &cli.App{
Name: "ricapi",
Commands: commands,
Before: func(c *cli.Context) error {
godotenv.Load(".env")
return nil
},
}
err := s.Run(os.Args)
if err != nil {
log.Fatal(err)
}
}

0
cmd/cli/digi/digi.go → cmd/digi/digi.go
View File

39
cmd/cli/plati/plati.go → cmd/plati/plati.go
View File

@ -7,16 +7,25 @@ import (
"log"
"os"
"strings"
"time"
"gitea.home.4it.me/dilap54/platiparser/gorm"
"gitea.home.4it.me/dilap54/platiparser/plati"
"github.com/joho/godotenv"
uuid "github.com/satori/go.uuid"
)
func init() {
godotenv.Load(".env")
}
func main() {
categories := openCategories("./categories.json").Content
categories = filterBySubstring("Gift", categories)
ctx := context.Background()
db := gorm.GetDB()
platiCli := plati.New()
// printNames(categories)
@ -27,10 +36,40 @@ func main() {
if err != nil {
log.Fatal(err)
}
if len(goods) == 0 {
continue
}
gormGoods := convertGoodsToGorm(c, goods)
if err := db.Create(gormGoods).Error; err != nil {
log.Fatal(err)
}
json.NewEncoder(os.Stdout).Encode(goods)
}
}
func convertGoodsToGorm(cat *plati.Category, goods []*plati.Good) []*gorm.Good {
out := make([]*gorm.Good, 0, len(goods))
for _, g := range goods {
gormGood := gorm.Good{
ID: uuid.NewV4().String(),
Name: g.Name,
IDC: cat.ID,
IDR: cat.ParentID,
Goodlink: g.GoodLink,
Seller: g.Seller,
Sellerlink: g.SellerLink,
Sellerrating: g.SellerRating,
Sold: g.Sold,
Price: g.Price,
CreatedAt: time.Now(),
}
out = append(out, &gormGood)
}
return out
}
func filterBySubstring(substring string, categories plati.Categories) plati.Categories {
out := make(plati.Categories, 0)
for _, c := range categories {

26
go.mod
View File

@ -3,7 +3,27 @@ module gitea.home.4it.me/dilap54/platiparser
go 1.21
require (
github.com/PuerkitoBio/goquery v1.8.1 // indirect
github.com/andybalholm/cascadia v1.3.1 // indirect
golang.org/x/net v0.7.0 // indirect
github.com/PuerkitoBio/goquery v1.8.1
github.com/joho/godotenv v1.5.1
github.com/pressly/goose v2.7.0+incompatible
github.com/urfave/cli/v2 v2.26.0
gorm.io/driver/postgres v1.5.4
gorm.io/gorm v1.25.5
)
require (
github.com/andybalholm/cascadia v1.3.1 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
github.com/jackc/pgx/v5 v5.4.3 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/satori/go.uuid v1.2.0 // indirect
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
golang.org/x/crypto v0.14.0 // indirect
golang.org/x/net v0.10.0 // indirect
golang.org/x/text v0.13.0 // indirect
)

51
go.sum
View File

@ -2,16 +2,55 @@ github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAc
github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ=
github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.4.3 h1:cxFyXhxlvAifxnkKKdlxv8XqUf59tDlYjnV5YYfsJJY=
github.com/jackc/pgx/v5 v5.4.3/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSljmHRA=
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pressly/goose v2.7.0+incompatible h1:PWejVEv07LCerQEzMMeAtjuyCKbyprZ/LBa6K5P0OCQ=
github.com/pressly/goose v2.7.0+incompatible/go.mod h1:m+QHWCqxR3k8D9l7qfzuC/djtlfzxr34mozWDYEu1z8=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/urfave/cli/v2 v2.26.0 h1:3f3AMg3HpThFNT4I++TKOejZO8yU55t3JnnSr4S4QEI=
github.com/urfave/cli/v2 v2.26.0/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@ -29,7 +68,17 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gorm.io/driver/postgres v1.5.4 h1:Iyrp9Meh3GmbSuyIAGyjkN+n9K+GHX9b9MqsTL4EJCo=
gorm.io/driver/postgres v1.5.4/go.mod h1:Bgo89+h0CRcdA33Y6frlaHHVuTdOf87pmyzwW9C/BH0=
gorm.io/gorm v1.25.5 h1:zR9lOiiYf09VNh5Q1gphfyia1JpiClIWG9hQaxB/mls=
gorm.io/gorm v1.25.5/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=

20
gorm/errors.go Normal file
View File

@ -0,0 +1,20 @@
package gorm
import (
"errors"
"gorm.io/gorm"
)
var ErrRecordNotFound = gorm.ErrRecordNotFound
var ZeroRowsAffected = errors.New("zero rows was affected")
func ErrOrZeroAffected(resp *gorm.DB) error {
if resp.Error != nil {
return resp.Error
}
if resp.RowsAffected == 0 {
return ZeroRowsAffected
}
return nil
}

21
gorm/good.go Normal file
View File

@ -0,0 +1,21 @@
package gorm
import "time"
type Good struct {
ID string
Name string
IDC int
IDR int
Goodlink string
Seller string
Sellerlink string
Sellerrating int
Sold int
Price float64
CreatedAt time.Time
}
func (Good) TableName() string {
return "goods"
}

56
gorm/gorm.go Normal file
View File

@ -0,0 +1,56 @@
package gorm
import (
"log"
"os"
"sync"
"time"
"gorm.io/driver/postgres"
"gorm.io/gorm"
gormlogger "gorm.io/gorm/logger"
)
var db *gorm.DB
var mu = sync.Mutex{}
func GetDB() *gorm.DB {
if db != nil {
return db
}
mu.Lock()
defer mu.Unlock()
if db == nil {
var err error
logLevel := gormlogger.Warn
if os.Getenv("DB_DEBUG") == "true" {
logLevel = gormlogger.Info
}
db, err = gorm.Open(postgres.Open(os.Getenv("POSTGRESQL_DSN")), &gorm.Config{
Logger: gormlogger.New(
log.Default(),
gormlogger.Config{
SlowThreshold: time.Second, // Slow SQL threshold
LogLevel: logLevel, // Log level
Colorful: false, // Disable color
},
),
})
if err != nil {
log.Fatal(err)
}
sqlDB, err := db.DB()
if err != nil {
log.Fatalf("failed to get sql db: %s", err.Error())
return nil
}
sqlDB.SetConnMaxIdleTime(3 * time.Minute)
sqlDB.SetConnMaxLifetime(10 * time.Minute)
sqlDB.SetMaxIdleConns(1)
sqlDB.SetMaxOpenConns(10)
}
return db
}

5
gorm/type.go Normal file
View File

@ -0,0 +1,5 @@
package gorm
import "gorm.io/gorm"
type DB = gorm.DB

16
migrations/20231220222206_init.sql Normal file
View File

@ -0,0 +1,16 @@
-- +goose Up
-- +goose StatementBegin
CREATE TABLE categories (
id INTEGER NOT NULL PRIMARY KEY,
parent_id INTEGER NOT NULL DEFAULT 0,
title VARCHAR NOT NULL DEFAULT '',
name VARCHAR NOT NULL DEFAULT '',
flat_name VARCHAR NOT NULL DEFAULT '',
level INTEGER NOT NULL DEFAULT 0
);
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
DROP TABLE categories;
-- +goose StatementEnd

21
migrations/20231220223134_goods.sql Normal file
View File

@ -0,0 +1,21 @@
-- +goose Up
-- +goose StatementBegin
CREATE TABLE goods (
id VARCHAR NOT NULL PRIMARY KEY,
name VARCHAR NOT NULL DEFAULT '',
id_c INTEGER NOT NULL,
id_r INTEGER NOT NULL,
goodlink VARCHAR NOT NULL DEFAULT '',
seller VARCHAR NOT NULL DEFAULT '',
sellerlink VARCHAR NOT NULL DEFAULT '',
sellerrating INTEGER NOT NULL DEFAULT 0,
sold INTEGER NOT NULL DEFAULT 0,
price DECIMAL(15, 2) NOT NULL DEFAULT 0,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
DROP TABLE goods;
-- +goose StatementEnd

29
out.log Normal file
View File

File diff suppressed because one or more lines are too long