From f5f23f8a2783f887b2195cc077b7f525b98e3362 Mon Sep 17 00:00:00 2001 From: Daniel Goc Date: Mon, 23 Mar 2026 16:42:54 +0100 Subject: [PATCH] new fields in meili indexing --- .env | 4 ++ .../web/api/restricted/listProducts.go | 8 ++-- .../web/api/restricted/meiliSearch.go | 14 +++---- app/delivery/web/api/restricted/menu.go | 8 ++-- app/model/productDescription.go | 28 ++++++++++--- .../productDescriptionRepo.go | 33 +++++++++++++++ app/service/meiliService/blank.json | 8 ++++ app/service/meiliService/meiliService.go | 41 ++++++------------- .../productDescriptionService.go | 2 +- 9 files changed, 93 insertions(+), 53 deletions(-) create mode 100644 app/service/meiliService/blank.json diff --git a/.env b/.env index 8fe69fd..bc636c1 100644 --- a/.env +++ b/.env @@ -25,6 +25,9 @@ AUTH_REFRESH_EXPIRATION=604800 MEILISEARCH_URL=http://localhost:7700 MEILISEARCH_API_KEY=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +# OpenAI +OPENAI_KEY=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 + # Google Translate Client GOOGLE_APPLICATION_CREDENTIALS=./google-cred.json GOOGLE_CLOUD_PROJECT_ID=translation-343517 @@ -33,6 +36,7 @@ GOOGLE_CLOUD_PROJECT_ID=translation-343517 OAUTH_GOOGLE_CLIENT_ID=331979954218-9vrpe08oqhhcgj6bvu6d4lds0dt630m9.apps.googleusercontent.com OAUTH_GOOGLE_CLIENT_SECRET=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 OAUTH_GOOGLE_REDIRECT_URL=http://localhost:3000/api/v1/public/auth/google/callback + # Email Configuration (SMTP) # Set EMAIL_ENABLED=true to require email verification EMAIL_ENABLED=true diff --git a/app/delivery/web/api/restricted/listProducts.go b/app/delivery/web/api/restricted/listProducts.go index aeec22a..85317a6 100644 --- a/app/delivery/web/api/restricted/listProducts.go +++ b/app/delivery/web/api/restricted/listProducts.go @@ -1,8 +1,6 @@ package restricted import ( - "strconv" - "git.ma-al.com/goc_daniel/b2b/app/config" "git.ma-al.com/goc_daniel/b2b/app/service/listProductsService" "git.ma-al.com/goc_daniel/b2b/app/utils/i18n" @@ -52,13 +50,13 @@ func (h *ListProductsHandler) GetListing(c fiber.Ctx) error { // "override_currency": c.Query("override_currency", ""), // } - id_lang, err := strconv.Atoi(c.Cookies("lang_id", "2")) - if err != nil { + id_lang, ok := c.Locals("langID").(uint) + if !ok { return c.Status(responseErrors.GetErrorStatus(responseErrors.ErrBadAttribute)). JSON(response.Make(nullable.GetNil(""), 0, responseErrors.GetErrorCode(c, responseErrors.ErrBadAttribute))) } - listing, err := h.listProductsService.GetListing(uint(id_lang), paging, filters) + listing, err := h.listProductsService.GetListing(id_lang, paging, filters) if err != nil { return c.Status(responseErrors.GetErrorStatus(err)). JSON(response.Make(nullable.GetNil(""), 0, responseErrors.GetErrorCode(c, err))) diff --git a/app/delivery/web/api/restricted/meiliSearch.go b/app/delivery/web/api/restricted/meiliSearch.go index b0b5aeb..8a41381 100644 --- a/app/delivery/web/api/restricted/meiliSearch.go +++ b/app/delivery/web/api/restricted/meiliSearch.go @@ -1,8 +1,6 @@ package restricted import ( - "strconv" - "git.ma-al.com/goc_daniel/b2b/app/service/meiliService" "git.ma-al.com/goc_daniel/b2b/app/utils/i18n" "git.ma-al.com/goc_daniel/b2b/app/utils/nullable" @@ -32,13 +30,13 @@ func MeiliSearchHandlerRoutes(r fiber.Router) fiber.Router { } func (h *MeiliSearchHandler) CreateIndex(c fiber.Ctx) error { - id_lang, err := strconv.Atoi(c.Cookies("lang_id", "2")) - if err != nil { + id_lang, ok := c.Locals("langID").(uint) + if !ok { return c.Status(responseErrors.GetErrorStatus(responseErrors.ErrBadAttribute)). JSON(response.Make(nullable.GetNil(""), 0, responseErrors.GetErrorCode(c, responseErrors.ErrBadAttribute))) } - err = h.meiliService.CreateIndex(uint(id_lang)) + err := h.meiliService.CreateIndex(id_lang) if err != nil { return c.Status(responseErrors.GetErrorStatus(err)). JSON(response.Make(nullable.GetNil(""), 0, responseErrors.GetErrorCode(c, err))) @@ -49,13 +47,13 @@ func (h *MeiliSearchHandler) CreateIndex(c fiber.Ctx) error { } func (h *MeiliSearchHandler) Test(c fiber.Ctx) error { - id_lang, err := strconv.Atoi(c.Cookies("lang_id", "2")) - if err != nil { + id_lang, ok := c.Locals("langID").(uint) + if !ok { return c.Status(responseErrors.GetErrorStatus(responseErrors.ErrBadAttribute)). JSON(response.Make(nullable.GetNil(""), 0, responseErrors.GetErrorCode(c, responseErrors.ErrBadAttribute))) } - test, err := h.meiliService.Test(uint(id_lang)) + test, err := h.meiliService.Test(id_lang) if err != nil { return c.Status(responseErrors.GetErrorStatus(err)). JSON(response.Make(nullable.GetNil(""), 0, responseErrors.GetErrorCode(c, err))) diff --git a/app/delivery/web/api/restricted/menu.go b/app/delivery/web/api/restricted/menu.go index d20cd45..04a4233 100644 --- a/app/delivery/web/api/restricted/menu.go +++ b/app/delivery/web/api/restricted/menu.go @@ -1,8 +1,6 @@ package restricted import ( - "strconv" - "git.ma-al.com/goc_daniel/b2b/app/service/menuService" "git.ma-al.com/goc_daniel/b2b/app/utils/i18n" "git.ma-al.com/goc_daniel/b2b/app/utils/nullable" @@ -31,13 +29,13 @@ func MenuHandlerRoutes(r fiber.Router) fiber.Router { } func (h *MenuHandler) GetMenu(c fiber.Ctx) error { - id_lang, err := strconv.Atoi(c.Cookies("lang_id", "2")) - if err != nil { + id_lang, ok := c.Locals("langID").(uint) + if !ok { return c.Status(responseErrors.GetErrorStatus(responseErrors.ErrBadAttribute)). JSON(response.Make(nullable.GetNil(""), 0, responseErrors.GetErrorCode(c, responseErrors.ErrBadAttribute))) } - menu, err := h.menuService.GetMenu(uint(id_lang)) + menu, err := h.menuService.GetMenu(id_lang) if err != nil { return c.Status(responseErrors.GetErrorStatus(err)). JSON(response.Make(nullable.GetNil(""), 0, responseErrors.GetErrorCode(c, err))) diff --git a/app/model/productDescription.go b/app/model/productDescription.go index e03bbc1..d7c0e2e 100644 --- a/app/model/productDescription.go +++ b/app/model/productDescription.go @@ -19,10 +19,26 @@ type ProductDescription struct { Usage string `gorm:"column:usage;type:text" json:"usage" form:"usage"` } -type MeiliSearchProduct struct { - ProductID uint - Name string - Description string - DescriptionShort string - Usage string +type ProductRow struct { + IDProduct int `gorm:"column:id_product"` + IDShop int `gorm:"column:id_shop"` + Name string `gorm:"column:name"` + Active uint8 `gorm:"column:active"` + Reference string `gorm:"column:reference"` +} + +type MeiliSearchProduct struct { + ProductID uint `gorm:"column:id_product"` + Name string `gorm:"column:name"` + Active uint8 `gorm:"column:active"` + Price float64 `gorm:"column:price"` + Description string `gorm:"column:description"` + DescriptionShort string `gorm:"column:description_short"` + Usage string `gorm:"column:usage"` + EAN13 string `gorm:"column:ean13"` + Reference string `gorm:"column:reference"` + Width float64 `gorm:"column:width"` + Height float64 `gorm:"column:height"` + Depth float64 `gorm:"column:depth"` + Weight float64 `gorm:"column:weight"` } diff --git a/app/repos/productDescriptionRepo/productDescriptionRepo.go b/app/repos/productDescriptionRepo/productDescriptionRepo.go index ce4ad1b..fece6b8 100644 --- a/app/repos/productDescriptionRepo/productDescriptionRepo.go +++ b/app/repos/productDescriptionRepo/productDescriptionRepo.go @@ -12,6 +12,7 @@ type UIProductDescriptionRepo interface { GetProductDescription(productID uint, productLangID uint) (*model.ProductDescription, error) CreateIfDoesNotExist(productID uint, productLangID uint) error UpdateFields(productID uint, productLangID uint, updates map[string]string) error + GetMeiliProducts(id_lang uint) ([]model.MeiliSearchProduct, error) } type ProductDescriptionRepo struct{} @@ -73,3 +74,35 @@ func (r *ProductDescriptionRepo) UpdateFields(productID uint, productLangID uint return nil } + +// We assume that any user has access to all product descriptions +func (r *ProductDescriptionRepo) GetMeiliProducts(id_lang uint) ([]model.MeiliSearchProduct, error) { + var products []model.MeiliSearchProduct + + err := db.DB. + Select("pl.`usage` AS `usage`"). + Select(` + ps.id_product AS id_product, + pl.name AS name, + ps.active AS active, + ps.price AS price, + pl.description AS description, + pl.description_short AS description_short, + p.ean13 AS ean13, + p.reference AS reference, + p.width AS width, + p.height AS height, + p.depth AS depth, + p.weight AS weight + `). + Table("ps_product_shop AS ps"). + Joins("LEFT JOIN ps_product_lang AS pl ON ps.id_product = pl.id_product AND pl.id_shop = ? AND pl.id_lang = ?", constdata.SHOP_ID, id_lang). + Joins("LEFT JOIN ps_product AS p ON p.id_product = ps.id_product"). + Where("ps.id_shop = ?", constdata.SHOP_ID). + Scan(&products).Error + if err != nil { + return products, fmt.Errorf("database error: %w", err) + } + + return products, nil +} diff --git a/app/service/meiliService/blank.json b/app/service/meiliService/blank.json new file mode 100644 index 0000000..22446a1 --- /dev/null +++ b/app/service/meiliService/blank.json @@ -0,0 +1,8 @@ +{ + "products-openai": { + "source": "openAi", + "model": "text-embedding-3-small", + "apiKey": "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", + "documentTemplate": "{{doc.Name}} is equipment used for {{doc.Description | truncatewords: 20}}" + } +} \ No newline at end of file diff --git a/app/service/meiliService/meiliService.go b/app/service/meiliService/meiliService.go index 65847b6..196d38a 100644 --- a/app/service/meiliService/meiliService.go +++ b/app/service/meiliService/meiliService.go @@ -9,14 +9,14 @@ import ( "strings" "time" - "git.ma-al.com/goc_daniel/b2b/app/db" - "git.ma-al.com/goc_daniel/b2b/app/model" + "git.ma-al.com/goc_daniel/b2b/app/repos/productDescriptionRepo" constdata "git.ma-al.com/goc_daniel/b2b/app/utils/const_data" "github.com/meilisearch/meilisearch-go" ) type MeiliService struct { - meiliClient meilisearch.ServiceManager + productDescriptionRepo productDescriptionRepo.UIProductDescriptionRepo + meiliClient meilisearch.ServiceManager } func New() *MeiliService { @@ -29,57 +29,42 @@ func New() *MeiliService { ) return &MeiliService{ - meiliClient: client, + meiliClient: client, + productDescriptionRepo: productDescriptionRepo.New(), } } // ==================================== FOR SUPERADMIN ONLY ==================================== func (s *MeiliService) CreateIndex(id_lang uint) error { - var products []model.ProductDescription + products, err := s.productDescriptionRepo.GetMeiliProducts(id_lang) - err := db.DB. - Table("ps_product_lang"). - Where("id_shop = ? AND id_lang = ?", constdata.SHOP_ID, id_lang). - Scan(&products).Error - if err != nil { - return fmt.Errorf("database error: %w", err) - } - - var meiliProducts []model.MeiliSearchProduct for i := 0; i < len(products); i++ { - var nextMeiliProduct model.MeiliSearchProduct - - nextMeiliProduct.ProductID = products[i].ProductID - nextMeiliProduct.Name = products[i].Name - - nextMeiliProduct.Description, err = cleanHTML(products[i].Description) + products[i].Description, err = cleanHTML(products[i].Description) if err != nil { - fmt.Printf("nextMeiliProduct.Description: %v\n", nextMeiliProduct.Description) + fmt.Printf("products[i].Description: %v\n", products[i].Description) fmt.Printf("products[i].ProductID: %v\n", products[i].ProductID) fmt.Println("failed at description") fmt.Printf("err: %v\n", err) return err } - nextMeiliProduct.DescriptionShort, err = cleanHTML(products[i].DescriptionShort) + products[i].DescriptionShort, err = cleanHTML(products[i].DescriptionShort) if err != nil { - fmt.Printf("nextMeiliProduct.DescriptionShort: %v\n", nextMeiliProduct.DescriptionShort) + fmt.Printf("products[i].DescriptionShort: %v\n", products[i].DescriptionShort) fmt.Printf("products[i].ProductID: %v\n", products[i].ProductID) fmt.Println("failed at description short") fmt.Printf("err: %v\n", err) return err } - nextMeiliProduct.Usage, err = cleanHTML(products[i].Usage) + products[i].Usage, err = cleanHTML(products[i].Usage) if err != nil { - fmt.Printf("nextMeiliProduct.Usage: %v\n", nextMeiliProduct.Usage) + fmt.Printf("products[i].Usage: %v\n", products[i].Usage) fmt.Printf("products[i].ProductID: %v\n", products[i].ProductID) fmt.Println("failed at usage") fmt.Printf("err: %v\n", err) return err } - - meiliProducts = append(meiliProducts, nextMeiliProduct) } indexName := "meili_products_shop" + strconv.FormatInt(constdata.SHOP_ID, 10) + "_lang" + strconv.FormatInt(int64(id_lang), 10) @@ -89,7 +74,7 @@ func (s *MeiliService) CreateIndex(id_lang uint) error { SkipCreation: false, } - task, err := s.meiliClient.Index(indexName).AddDocuments(meiliProducts, docOptions) + task, err := s.meiliClient.Index(indexName).AddDocuments(products, docOptions) if err != nil { return fmt.Errorf("meili AddDocuments error: %w", err) } diff --git a/app/service/productDescriptionService/productDescriptionService.go b/app/service/productDescriptionService/productDescriptionService.go index df137ff..940ea41 100644 --- a/app/service/productDescriptionService/productDescriptionService.go +++ b/app/service/productDescriptionService/productDescriptionService.go @@ -70,7 +70,7 @@ func New() *ProductDescriptionService { log.Fatalf("productDescriptionService: cannot create Translation client: %v", err) } - openAIClient := openai.NewClient(option.WithAPIKey("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"), + openAIClient := openai.NewClient(option.WithAPIKey(os.Getenv("OPENAI_KEY")), option.WithHTTPClient(&http.Client{Timeout: 300 * time.Second})) // five minutes timeout return &ProductDescriptionService{