fix code

suggest-task/src/main/java/com/secoo/so/suggest/task/SuggestTask.java

@@ -204,24 +204,48 @@ public class SuggestTask {
         long batchSize = ConfigUtil.getLong("suggestTask.batchSize", 10000);
         int threadPoolSize = ConfigUtil.getInt("suggestTask.threadPoolSize", 10);
         ExecutorService execThreadPool = Executors.newFixedThreadPool(threadPoolSize);
+        int taskCount = 0;
         for (long startId = minId; startId <= maxId; startId = startId + batchSize) {
+            taskCount++;
             execThreadPool.submit(new SearchKeywordProcessTask(esSuggestKeywordMap, startId, startId + batchSize, startTime));
         }
         execThreadPool.shutdown();
         while (true) {
             if (execThreadPool.isTerminated()) {
-                log.info("所有的子线程都结束了, 关闭线程池成功");
+                log.info("所有的子线程任务 {}个 都结束了, 关闭线程池成功", taskCount);
                 break;
             }
             ObjectUtils.safeSleep(5000);
         }
 
+        // 过滤词
+        List<EsSuggestKeywordInfo> suggestKeywordInfoList = new ArrayList<>();
+        int processCount = 0;
+        int totalCount =  esSuggestKeywordMap.values().size();
+        for (EsSuggestKeywordInfo suggestKeywordInfo : esSuggestKeywordMap.values()) {
+            // 不过滤的suggest词，计算分值写es
+            if (!isFilterSuggestKeyword(suggestKeywordInfo)) {
+
+                // 转拼音
+                suggestKeywordInfo.setKeywordPinYin(PinYinUtils.changeToWithoutTonePinYin(suggestKeywordInfo.getKeyword(), ""));
+
+                // 保存es前执行标签清洗
+                cleanBeforeSaveToEs(suggestKeywordInfo);
+
+                suggestKeywordInfoList.add(suggestKeywordInfo);
+            }
+
+            processCount++;
+            if (processCount > 0 && processCount % 1000 == 0 || processCount == totalCount) {
+                log.info("keyword filter process: {} / {}", processCount, totalCount);
+            }
+        }
 
         // 保存到es
-//            saveSuggestKeywordToEs(new ArrayList<>(esSuggestKeywordMap.values()));
+//            saveSuggestKeywordToEs(suggestKeywordInfoList);
 
         // for test
-        saveSuggestKeywordToFile(new ArrayList<>(esSuggestKeywordMap.values()));
+        saveSuggestKeywordToFile(suggestKeywordInfoList);
     }
 
     /**
@@ -238,7 +262,6 @@ public class SuggestTask {
                         if (suggestKeywordInfo == null) {
                             suggestKeywordInfo = new EsSuggestKeywordInfo();
                             suggestKeywordInfo.setKeyword(keyword);
-                            suggestKeywordInfo.setKeywordPinYin(PinYinUtils.changeToWithoutTonePinYin(suggestKeywordInfo.getKeyword(), ""));
 
                             suggestKeywordInfo.setYearCount(searchKeywordInfo.getYearPv());
                             suggestKeywordInfo.setYearClickCount(searchKeywordInfo.getYearProductClickCount());
@@ -270,15 +293,6 @@ public class SuggestTask {
 
                         // 计算suggestKeyword权重等属性
                         processEsSuggestKeywordInfo(suggestKeywordInfo, searchKeywordInfo);
-
-                        // 不过滤的suggest词，计算分值写es
-                        if (!isFilterSuggestKeyword(suggestKeywordInfo)) {
-
-                            // 保存es前执行标签清洗
-                            cleanBeforeSaveToEs(suggestKeywordInfo);
-
-                            esSuggestKeywordMap.put(suggestKeywordInfo.getKeyword(), suggestKeywordInfo);
-                        }
                     }
                 }
             }
@@ -374,7 +388,7 @@ public class SuggestTask {
                 return;
             }
             List<String> tagList = StringUtils.splitToList(suggestKeywordInfo.getSuggestTags(), ",");
-            if (tagList.size() > maxTagSize) {
+            if (tagList != null && tagList.size() > maxTagSize) {
                 suggestKeywordInfo.setSuggestTags(StringUtils.join(CollectionUtils.subList(tagList, 0, maxTagSize), ","));
             }
         }

suggest-task/src/main/java/com/secoo/so/suggest/util/PinYinUtils.java

@@ -101,7 +101,7 @@ public class PinYinUtils {
         try {
             tempStr = PinyinHelper.convertToPinyinString(str, separator, PinyinFormat.WITHOUT_TONE);
         } catch (Exception e) {
-            log.error("changeToWithoutTonePinYin error", e);
+            log.error("changeToWithoutTonePinYin error: str:{}", str, e);
         }
         return tempStr;
     }