Merge branch 'TECH-search-fix' into 'master'

优化suggest算分逻辑

See merge request !3

suggest-task/src/main/java/com/secoo/so/suggest/entity/EsSuggestKeywordInfo.java

 package com.secoo.so.suggest.entity;
 
+import com.alibaba.fastjson.annotation.JSONField;
 import lombok.Data;
 
 import java.io.Serializable;
@@ -33,5 +34,24 @@ public class EsSuggestKeywordInfo implements Serializable {
     private String suggestTags;
     private Long updateTime;
 
-
+    @JSONField(serialize = false)
+    private Long monthPv;
+    @JSONField(serialize = false)
+    private Long monthProductClickCount;
+    @JSONField(serialize = false)
+    private Long monthAddCartCount;
+    @JSONField(serialize = false)
+    private Long monthUv;
+    @JSONField(serialize = false)
+    private Long monthProductClickUv;
+    @JSONField(serialize = false)
+    private Long monthAddCartUv;
+    @JSONField(serialize = false)
+    private Long weekAddCartCount;
+    @JSONField(serialize = false)
+    private Long weekUv;
+    @JSONField(serialize = false)
+    private Long weekProductClickUv;
+    @JSONField(serialize = false)
+    private Long weekAddCartUv;
 }

suggest-task/src/main/java/com/secoo/so/suggest/task/SuggestTask.java

@@ -289,6 +289,16 @@ public class SuggestTask {
         esSuggestKeywordInfo.setWeekCount(0L);
         esSuggestKeywordInfo.setWeekClickCount(0L);
         esSuggestKeywordInfo.setWeekCartCount(0L);
+        esSuggestKeywordInfo.setMonthPv(0L);
+        esSuggestKeywordInfo.setMonthProductClickCount(0L);
+        esSuggestKeywordInfo.setMonthAddCartCount(0L);
+        esSuggestKeywordInfo.setMonthUv(0L);
+        esSuggestKeywordInfo.setMonthProductClickUv(0L);
+        esSuggestKeywordInfo.setMonthAddCartUv(0L);
+        esSuggestKeywordInfo.setWeekAddCartCount(0L);
+        esSuggestKeywordInfo.setWeekUv(0L);
+        esSuggestKeywordInfo.setWeekProductClickUv(0L);
+        esSuggestKeywordInfo.setWeekAddCartUv(0L);
 
         SearchKeywordInfo searchKeywordInfo = new SearchKeywordInfo();
         searchKeywordInfo.setId(0L);
@@ -340,6 +350,17 @@ public class SuggestTask {
                             suggestKeywordInfo.setWeekClickCount(searchKeywordInfo.getWeekProductClickCount());
                             suggestKeywordInfo.setWeekCartCount(searchKeywordInfo.getWeekAddCartCount());
 
+                            suggestKeywordInfo.setMonthPv(searchKeywordInfo.getMonthPv());
+                            suggestKeywordInfo.setMonthProductClickCount(searchKeywordInfo.getMonthProductClickCount());
+                            suggestKeywordInfo.setMonthAddCartCount(searchKeywordInfo.getMonthAddCartCount());
+                            suggestKeywordInfo.setMonthUv(searchKeywordInfo.getMonthUv());
+                            suggestKeywordInfo.setMonthProductClickUv(searchKeywordInfo.getMonthProductClickUv());
+                            suggestKeywordInfo.setMonthAddCartUv(searchKeywordInfo.getMonthAddCartUv());
+                            suggestKeywordInfo.setWeekAddCartCount(searchKeywordInfo.getWeekAddCartCount());
+                            suggestKeywordInfo.setWeekUv(searchKeywordInfo.getWeekUv());
+                            suggestKeywordInfo.setWeekProductClickUv(searchKeywordInfo.getWeekProductClickUv());
+                            suggestKeywordInfo.setWeekAddCartUv(searchKeywordInfo.getWeekAddCartUv());
+
                             suggestKeywordInfo.setSuggestTags(searchKeywordInfo.getPrepareTags());
                             suggestKeywordInfo.setKeywordVersion(searchKeywordInfo.getPDay());
                             suggestKeywordInfo.setUpdateTime(startTime);
@@ -352,6 +373,18 @@ public class SuggestTask {
                             suggestKeywordInfo.setWeekCount(suggestKeywordInfo.getWeekCount() + searchKeywordInfo.getWeekPv());
                             suggestKeywordInfo.setWeekClickCount(suggestKeywordInfo.getWeekClickCount() + searchKeywordInfo.getWeekProductClickCount());
                             suggestKeywordInfo.setWeekCartCount(suggestKeywordInfo.getWeekCartCount() + searchKeywordInfo.getWeekAddCartCount());
+
+                            suggestKeywordInfo.setMonthPv(suggestKeywordInfo.getMonthPv() + searchKeywordInfo.getMonthPv());
+                            suggestKeywordInfo.setMonthProductClickCount(suggestKeywordInfo.getMonthProductClickCount() + searchKeywordInfo.getMonthProductClickCount());
+                            suggestKeywordInfo.setMonthAddCartCount(suggestKeywordInfo.getMonthAddCartCount() + searchKeywordInfo.getMonthAddCartCount());
+                            suggestKeywordInfo.setMonthUv(suggestKeywordInfo.getMonthUv() + searchKeywordInfo.getMonthUv());
+                            suggestKeywordInfo.setMonthProductClickUv(suggestKeywordInfo.getMonthProductClickUv() + searchKeywordInfo.getMonthProductClickUv());
+                            suggestKeywordInfo.setMonthAddCartUv(suggestKeywordInfo.getMonthAddCartUv() + searchKeywordInfo.getMonthAddCartUv());
+                            suggestKeywordInfo.setWeekAddCartCount(suggestKeywordInfo.getWeekAddCartCount() + searchKeywordInfo.getWeekAddCartCount());
+                            suggestKeywordInfo.setWeekUv(suggestKeywordInfo.getWeekUv() + searchKeywordInfo.getWeekUv());
+                            suggestKeywordInfo.setWeekProductClickUv(suggestKeywordInfo.getWeekProductClickUv() + searchKeywordInfo.getWeekProductClickUv());
+                            suggestKeywordInfo.setWeekAddCartUv(suggestKeywordInfo.getWeekAddCartUv() + searchKeywordInfo.getWeekAddCartUv());
+
                             if (StringUtils.isBlank(suggestKeywordInfo.getSuggestTags()) && "null".equalsIgnoreCase(suggestKeywordInfo.getSuggestTags())) {
                                 suggestKeywordInfo.setSuggestTags(suggestKeywordInfo.getSuggestTags());
                             }
@@ -545,7 +578,7 @@ public class SuggestTask {
         }
 
         calculateWordRank(suggestKeywordInfo);
-        calculateWordABRank(suggestKeywordInfo, searchKeywordInfo);
+        calculateWordABRank(suggestKeywordInfo);
         addNewScoreIfNewHotWord(suggestKeywordInfo);
 
     }
@@ -586,33 +619,34 @@ public class SuggestTask {
         return wordRank;
     }
 
-    public static Double calculateWordABRank(EsSuggestKeywordInfo suggestKeywordInfo, SearchKeywordInfo searchKeywordInfo) {
+
+    public static Double calculateWordABRank(EsSuggestKeywordInfo suggestKeywordInfo) {
 
         // 月点击加购率
-        Double monthClickRatio = CalculateUtils.calculateRatio(searchKeywordInfo.getMonthProductClickUv().intValue(), searchKeywordInfo.getMonthUv().intValue());
-        Double monthCartRatio = CalculateUtils.calculateRatio(searchKeywordInfo.getMonthAddCartUv().intValue(), searchKeywordInfo.getMonthUv().intValue());
+        Double monthClickRatio = CalculateUtils.calculateRatio(suggestKeywordInfo.getMonthProductClickUv().intValue(), suggestKeywordInfo.getMonthUv().intValue());
+        Double monthCartRatio = CalculateUtils.calculateRatio(suggestKeywordInfo.getMonthAddCartUv().intValue(), suggestKeywordInfo.getMonthUv().intValue());
 
         // 周点击加购率(和A相比, count 换成了uv)
-        Double weekClickRatioNew = CalculateUtils.calculateRatio(searchKeywordInfo.getWeekProductClickUv().intValue(), searchKeywordInfo.getWeekUv().intValue());
-        Double weekCartRatioNew = CalculateUtils.calculateRatio(searchKeywordInfo.getWeekAddCartUv().intValue(), searchKeywordInfo.getWeekUv().intValue());
+        Double weekClickRatioNew = CalculateUtils.calculateRatio(suggestKeywordInfo.getWeekProductClickUv().intValue(), suggestKeywordInfo.getWeekUv().intValue());
+        Double weekCartRatioNew = CalculateUtils.calculateRatio(suggestKeywordInfo.getWeekAddCartUv().intValue(), suggestKeywordInfo.getWeekUv().intValue());
 
         // 月点击
-        if (searchKeywordInfo.getMonthProductClickUv() != 0 && searchKeywordInfo.getMonthUv() != 0) {
+        if (suggestKeywordInfo.getMonthProductClickUv() != 0 && suggestKeywordInfo.getMonthUv() != 0) {
             monthClickRatio *= 1.5;
         }
 
         // 月加购，加权
-        if (searchKeywordInfo.getMonthAddCartUv() != 0 && searchKeywordInfo.getMonthUv() != 0) {
+        if (suggestKeywordInfo.getMonthAddCartUv() != 0 && suggestKeywordInfo.getMonthUv() != 0) {
             monthCartRatio *= 3;
         }
 
         // 周点击，加权
-        if (searchKeywordInfo.getWeekProductClickUv() != 0 && searchKeywordInfo.getWeekUv() != 0) {
+        if (suggestKeywordInfo.getWeekProductClickUv() != 0 && suggestKeywordInfo.getWeekUv() != 0) {
             weekClickRatioNew *= 2;
         }
 
         // 周加购，加权
-        if (searchKeywordInfo.getWeekAddCartUv() != 0 && searchKeywordInfo.getWeekUv() != 0) {
+        if (suggestKeywordInfo.getWeekAddCartUv() != 0 && suggestKeywordInfo.getWeekUv() != 0) {
             weekCartRatioNew *= 3;
         }
 
@@ -622,25 +656,25 @@ public class SuggestTask {
         wordABRank += 3000 * CalculateUtils.calculateLengthFactor(StringUtils.getByteLength(suggestKeywordInfo.getKeyword()));
 
         // 月数量因子
-        wordABRank += 2000 * CalculateUtils.calculateCountFactor(searchKeywordInfo.getMonthUv().intValue(), 4);
+        wordABRank += 2000 * CalculateUtils.calculateCountFactor(suggestKeywordInfo.getMonthUv().intValue(), 4);
         // 周数量因子
-        wordABRank += 2000 * CalculateUtils.calculateCountFactor(searchKeywordInfo.getWeekUv().intValue(), 52);
+        wordABRank += 2000 * CalculateUtils.calculateCountFactor(suggestKeywordInfo.getWeekUv().intValue(), 52);
         // 年数量因子
         wordABRank += 2000 * CalculateUtils.calculateCountFactor(suggestKeywordInfo.getYearCount(), 1);
 
         // 点击
         // 月点击率因子
-        wordABRank += 3000 * CalculateUtils.calculateRatioFactor(monthClickRatio, searchKeywordInfo.getMonthProductClickUv().intValue());
+        wordABRank += 3000 * CalculateUtils.calculateRatioFactor(monthClickRatio, suggestKeywordInfo.getMonthProductClickUv().intValue() / 4);
         // 周点击率因子
-        wordABRank += 3000 * CalculateUtils.calculateRatioFactor(weekClickRatioNew, searchKeywordInfo.getWeekUv().intValue());
+        wordABRank += 3000 * CalculateUtils.calculateRatioFactor(weekClickRatioNew, suggestKeywordInfo.getWeekProductClickUv().intValue());
 
         // 加购
         // 年加购率因子
         wordABRank += 3000 * CalculateUtils.calculateRatioFactor(suggestKeywordInfo.getYearCartRatio().doubleValue(), suggestKeywordInfo.getYearCartCount());
         // 月加购率因子
-        wordABRank += 3000 * CalculateUtils.calculateRatioFactor(monthCartRatio, searchKeywordInfo.getMonthUv().intValue());
+        wordABRank += 3000 * CalculateUtils.calculateRatioFactor(monthCartRatio, suggestKeywordInfo.getMonthAddCartCount().intValue() * 12);
         // 周加购率因子
-        wordABRank += 3000 * CalculateUtils.calculateRatioFactor(weekCartRatioNew, searchKeywordInfo.getWeekUv().intValue());
+        wordABRank += 3000 * CalculateUtils.calculateRatioFactor(weekCartRatioNew, suggestKeywordInfo.getWeekAddCartCount().intValue() * 52);
 
         if (suggestKeywordInfo.getIsBrand()) {
             wordABRank *= 1.8;

suggest-task/src/main/java/com/secoo/so/suggest/util/CalculateUtils.java

@@ -21,19 +21,19 @@ public class CalculateUtils {
     public static Double calculateRatioFactor(Double ratio, Integer count) {
         Double rank = 1.0;
         if (count > 1 && count < 10) {
-            rank = 1.2;
+            rank = 1.02;
         } else if (count >= 10 && count < 20) {
-            rank = 1.4;
+            rank = 1.05;
         } else if (count >= 20 && count < 50) {
-            rank = 1.6;
+            rank = 1.08;
         } else if (count >= 50 && count < 100) {
-            rank = 1.8;
+            rank = 1.1;
         } else if (count >= 100 && count < 200) {
-            rank = 2.0;
+            rank = 1.2;
         } else if (count >= 200 && count < 500) {
-            rank = 2.2;
+            rank = 1.3;
         } else if (count >= 500) {
-            rank = 2.5;
+            rank = 1.5;
         }
         //根据搜索转化率，转换为热度因子
         return Math.log10(Math.sqrt(ratio + 10)) * rank;

suggest-task/src/test/java/com/secoo/so/suggest/TestCode.java

@@ -23,13 +23,38 @@ import java.util.List;
 public class TestCode {
 
 
+    public static void main(String[] args) throws Exception {
+        List<String> esLines = FileUtils.readLines(new File("C:\\Users\\secoo\\Desktop\\suggest-index-20220720144919.json"));
+        EsClient esClient = EsClient.buildEsClient("http://127.0.0.1:9200", "", "");
+
+        int count = 0;
+        List<EsObject> esList = new ArrayList<>();
+        for (String esLine : esLines) {
+            EsSuggestKeywordInfo esSuggestKeywordInfo = JSON.parseObject(esLine, EsSuggestKeywordInfo.class);
+            if (esSuggestKeywordInfo != null && StringUtils.isNotBlank(esSuggestKeywordInfo.getKeyword())) {
+                count++;
+                esList.add(new EsObject(StringUtils.md5(esSuggestKeywordInfo.getKeyword()), esSuggestKeywordInfo));
+            }
+            if (count == 1000) {
+                esClient.batch("search_suggest_index", "search_suggest_type", esList);
+                count = 0;
+                esList = new ArrayList<>();
+            }
+        }
+
+        if (esList.size() > 0) {
+            esClient.batch("search_suggest_index", "search_suggest_type", esList);
+        }
+        esClient.close();
+    }
+
     public static void main7(String[] args) throws Exception {
-     System.out.println(StringUtils.dbc2Sbc("模糊字母"));
-     System.out.println(PinYinUtils.convertToSimplifiedChinese("模糊字母"));
+        System.out.println(StringUtils.dbc2Sbc("模糊字母"));
+        System.out.println(PinYinUtils.convertToSimplifiedChinese("模糊字母"));
     }
 
 
-    public static void main(String[] args) throws Exception {
+    public static void main6(String[] args) throws Exception {
 
         List<String> esLines = FileUtils.readLines(new File("d:\\suggest-es.json"));
         List<String> newLines = FileUtils.readLines(new File("d:\\suggest-index-keyword-20220209142219.txt"));
@@ -50,7 +75,6 @@ public class TestCode {
 
     }
 
-
     public static void main5(String[] args) throws Exception {
         EsClient esClient = EsClient.buildEsClient("http://yunhead.siku.cn", "search", "search5z0NvEn1D");
         BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();