Skip to content

S
suggest-task
Commit 9bc3ae0b by 徐鹏
Options

Merge branch 'suggest-task-java' into 'master' 

Suggest task java

See merge request !1
parents 60d8f5e5 611e1909
Showing with 8292 additions and 2 deletions
.gitignore
...@@ -2,4 +2,47 @@ ...@@ -2,4 +2,47 @@
main/info.log main/info.log
main/test.go main/test.go
main/suggest-task main/suggest-task
main/suggest-task.exe main/suggest-task.exe
\ No newline at end of file
# Compiled class file
**/target
**/*.class
**/.classpath
**/.settings
**/.project
**/.idea
**/.vscode
**/*.iml
**/.DS_Store
**/node_modules
**/dist
*.suo
*.ntvs*
*.njsproj
*.sln
*.sublime-project
*.sublime-workspace
**/coverage
**/test/unit/coverage/
**/test/e2e/reports/
# Log file
*.log
**/*.log
# BlueJ files
*.ctxt
# Mobile Tools for Java (J2ME)
.mtj.tmp/
# Package Files #
*.jar
*.war
*.ear
*.zip
*.tar.gz
*.rar
# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
**/hs_err_pid*
\ No newline at end of file
suggest-task/assembly.xml 0 → 100644
<assembly>
<id>dependency</id>
<formats>
<format>jar</format>
</formats>
<!-- 压缩包下是否生成和项目名相同的根目录-->
<includeBaseDirectory>false</includeBaseDirectory>
<dependencySets>
<dependencySet>
<!-- 是否把当前项目的输出jar包并使用，true则会把当前项目输出为jar包到输出目录,false不输出 -->
<useProjectArtifact>false</useProjectArtifact>
<!-- 是否解压依赖包 -->
<unpack>true</unpack>
<scope>system</scope>
</dependencySet>
<dependencySet>
<!-- 是否解压依赖包 -->
<unpack>true</unpack>
<!-- 将scope为runtime的依赖包打包 -->
<scope>runtime</scope>
<excludes>
</excludes>
<includes>
</includes>
</dependencySet>
</dependencySets>
<fileSets>
<fileSet>
<directory>${project.build.outputDirectory}</directory>
<outputDirectory>/</outputDirectory>
</fileSet>
</fileSets>
</assembly>
\ No newline at end of file
suggest-task/bin/clear_access_log.sh 0 → 100644
#! /bin/bash
# zyc
# 删除 suggest 服务 7天前的 access_log 日志
# 因为 today 取的当前时间, fileDate 取得是文件创建当天0:00 的时间,所以不确保是整整 24*7 个小时
. /etc/profile
today=`date +%s`
basepath="/data/secoo_tomcat/javaapp/so-suggest-rest/logs"
for file in `ls $basepath`
do
if [[ $file == access* ]]; then
echo $file
fileDate=`date +%s -d ${file:0-14:10}`
echo $fileDate
timePast=$(($today -$fileDate))
if [[ $timePast -gt 3600*24*7 ]] ; then
rm -f $basepath"/"$file
fi
fi
done
suggest-task/bin/clear_offline_log.sh 0 → 100644
basePath="/data/crontab/suggest/logs"
cd $basePath
size=`ls -l| grep "service.log"|awk '{ print $5}'`
if [[ "$size" -gt $((80*1024*1024)) ]] ; then
cp service.log service-`date +%Y-%m-%d`.log
# 删除日志
rm -f service.log
# 删除七天前的备份日志
rm -f service-`date +%Y-%m-%d -d "-7 days"`.log
fi
suggest-task/bin/crontab.txt 0 → 100644
# 清理离线任务的日志
#48 9 * * * /data/crontab/suggest/clear_offline_log.sh >> /data/crontab/suggest/logs/clear.log 2>&1
# 清理suggest 服务的 access_log 日志
#0 0 */2 * * /data/crontab/suggest/clear_access_log.sh >> /data/crontab/suggest/logs/clear_access.log 2>&1
# suggest 离线索引任务
32 6,9 * * * /data/crontab/suggest/start_suggest_task.sh >> /data/crontab/suggest/logs/service.log 2>&1
suggest-task/bin/delete_old_version.py 0 → 100644
import requests
DEFAULT_HEADERS = {
"platform-type": "0",
"device-id": "c79d1dfeb84c4500_38864DF8:9009:4FB9:83F3:B3FAAE91606C",
"app-ver": "6.0.18",
'User-Agent': 'Secoo-iPhone/6.0.12 (iPhone; iOS 11.4.1; Scale/2.00)',
'Authorization': 'Basic c2VhcmNoOnNlYXJjaDV6ME52RW4xRA==',
'Content-Type': 'application/json'
}
def getMostDocTimestamp():
body = {
"size":0,
"aggs": {
"updateTimeAgg": {
"terms": {
"field": "updateTime"
}
}
}
}
resp = requests.post('http://yunhead.siku.cn/search_suggest_index/search_suggest_type/_search',
headers=DEFAULT_HEADERS,
json=body)
res_data = resp.json()
buckets = res_data['aggregations']['updateTimeAgg']['buckets']
most_doc_timestamp = -1
most_doc_count = 0
for bucket in buckets:
if bucket['doc_count'] > most_doc_count:
most_doc_timestamp = bucket['key']
most_doc_count = bucket['doc_count']
return most_doc_timestamp
def delNoneMostDocs(timestamp):
most_doc_count = checkMostDocCountByUpdateTime(timestamp)
print('check most doc count ' + str(most_doc_count))
if most_doc_count < 300000:
return
body = {
"query":{
"bool": {
"must_not": {
"term": {
"updateTime": timestamp
}
}
}
}
}
resp = requests.post('http://yunhead.siku.cn/search_suggest_index/search_suggest_type/_delete_by_query',
headers=DEFAULT_HEADERS,
json=body)
print(resp.text)
def checkMostDocCountByUpdateTime(timestamp):
body = {
"query":{
"bool": {
"must": {
"term": {
"updateTime": timestamp
}
}
}
}
}
resp = requests.post('http://yunhead.siku.cn/search_suggest_index/search_suggest_type/_search',
headers=DEFAULT_HEADERS,
json=body)
return resp.json()['hits']['total']
def start():
most_doc_timestamp = getMostDocTimestamp()
delNoneMostDocs(most_doc_timestamp)
if __name__ == "__main__":
start()
suggest-task/bin/notify_util.py 0 → 100644
# -*- coding:utf-8 -*-
import json
import requests
import os
import sys
import datetime
from optparse import OptionParser
def secooSendFeiShu(phones, title, content):
phone_list = phones.strip().split(',')
phone_list = map(lambda x: x.strip(), phone_list)
now_time = datetime.datetime.now()
date_str = datetime.datetime.strftime(now_time, "%Y-%m-%d %H:%M:%S")
content += ('\n' + date_str)
data = {
"phones": phone_list,
"title": title,
"body": [content.strip()]
}
# host = 'http://apims.siku.cn/mock/303/user/sendToUser'
host = 'http://matrix-inform.secoolocal.com/user/sendToUser'
headers = {"Content-Type": "application/json"}
print('req:', data)
rep = requests.post(host, json.dumps(data), headers=headers)
print('rep:', rep.content)
return rep.content
def get_option_parser():
usage = "usage: %prog [options] arg1 arg2"
parser = OptionParser(usage=usage)
parser.add_option("-p", "--phones", dest="phones", action="store", type="string")
parser.add_option("-t", "--title", dest="title", action="store", type="string")
parser.add_option("-c", "--content", dest="content", action="store", type="string")
return parser
if __name__ == '__main__':
# reload(sys)
# sys.setdefaultencoding('utf-8')
optParser = get_option_parser()
options, args = optParser.parse_args(sys.argv[1:])
print(options)
if options.phones is None or options.title is None or options.content is None:
optParser.print_help()
sys.exit(1)
else:
secooSendFeiShu(options.phones, options.title, options.content)
print('Done.')
suggest-task/bin/start_suggest_task.sh 0 → 100644
cd /data/crontab/suggest/
source /etc/profile
today=`date "+%Y-%m-%d"`
# 获取新日志
rm service.log
rm service.log.1
scp client1.secoo-inc.com:/data/hdfs/check_no_result/logs/service.log .
scp client1.secoo-inc.com:/data/hdfs/check_no_result/logs/service.log.1 .
cat ./service.log.1 >> service.log
#java -cp $CLASSPATH:./get_no_result/* com.secoo.so.searchword.task.LogExtractor ./service.log $today no_result_word > /data/pssmaster/corpus_set/suggest_corpus/sensitive/no_result_all.txt
#java -cp $CLASSPATH:./get_no_result/* com.secoo.so.searchword.task.LogExtractor ./service.log $today europe_word > /data/pssmaster/corpus_set/suggest_corpus/europe_word/europe_word.txt
cat ./service.log |awk -F'no result word:::' '{print $2}' | sort | uniq > /data/pssmaster/corpus_set/suggest_corpus/sensitive/no_result_all.txt
no_result_all_count=`wc -l /data/pssmaster/corpus_set/suggest_corpus/sensitive/no_result_all.txt`
echo "no result all count $no_result_all_count"
#go script
#./suggest-task >> /data/crontab/suggest/logs/message.log 2>&1
#java
./suggest-task.sh
if [[ $? -ne 0 ]] ; then
echo "error happened"
python notify_util.py -p "13426233960" -t "失败:es提示词更新" -c ""
else
python notify_util.py -p "13426233960" -t "es提示词更新成功" -c ""
python delete_old_version.py
fi
suggest-task/bin/suggest-task-save-to-file.sh 0 → 100644
#!/usr/bin/env bash
java -Xms2048m -Xmx4096m -Dsuggest.saveToFile=true -cp so-suggest-task-1.0-SNAPSHOT.jar com.secoo.so.suggest.task.SuggestTask
suggest-task/bin/suggest-task.sh 0 → 100644
#!/usr/bin/env bash
java -Xms2048m -Xmx4096m -cp so-suggest-task-1.0-SNAPSHOT.jar com.secoo.so.suggest.task.SuggestTask
suggest-task/pom.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.secoo.so</groupId>
<artifactId>so-suggest-task</artifactId>
<version>1.0-SNAPSHOT</version>
<name>so-suggest-task</name>
<profiles>
<profile>
<id>prod</id>
<build>
<resources>
<resource>
<directory>src/main/profiles/prod</directory>
</resource>
</resources>
</build>
</profile>
<profile>
<id>test</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<build>
<resources>
<resource>
<directory>src/main/profiles/test</directory>
</resource>
</resources>
</build>
</profile>
</profiles>
<dependencies>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.36</version>
</dependency>
<dependency>
<groupId>commons-dbcp</groupId>
<artifactId>commons-dbcp</artifactId>
<version>1.4</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.16.6</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.12</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.12</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.5</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.alibaba/fastjson -->
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.58</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpasyncclient</artifactId>
<version>4.1.1</version>
</dependency>
<dependency>
<groupId>com.secoo.search.third-patry</groupId>
<artifactId>third-patry-jpinyin</artifactId>
<version>1.1.8</version>
<scope>system</scope>
<systemPath>${project.basedir}/lib/third-patry-jpinyin-1.1.8.jar</systemPath>
</dependency>
<!-- es -->
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
<version>6.4.2</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>transport</artifactId>
<version>6.4.2</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>6.4.2</version>
</dependency>
</dependencies>
<build>
<resources>
<resource>
<directory>src/main/resources</directory>
</resource>
</resources>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.6.0</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
<encoding>UTF-8</encoding>
</configuration>
</plugin>
<!-- 打成 withDependencies jar 包-->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.6</version>
<configuration>
<!-- not append assembly id in release file name -->
<appendAssemblyId>false</appendAssemblyId>
<descriptors>
<descriptor>assembly.xml</descriptor>
</descriptors>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
suggest-task/src/main/java/com/secoo/so/suggest/config/ConfigUtil.java 0 → 100644
package com.secoo.so.suggest.config;
import lombok.extern.slf4j.Slf4j;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.Map;
import java.util.Properties;
@Slf4j
public class ConfigUtil {
private static final String CONFIG_FILE = "config.properties";
private static ConfigUtil INST;
private Properties prop;
private ConfigUtil() {
prop = load();
}
public synchronized static void init() {
ConfigUtil configUtil = new ConfigUtil();
INST = configUtil;
}
private Properties load() {
Properties tmp = new Properties();
try {
log.info(this.getClass().getClassLoader().getResource(CONFIG_FILE).getPath());
tmp.load(new InputStreamReader(this.getClass().getClassLoader().getResourceAsStream(CONFIG_FILE), "UTF-8"));
} catch (IOException e) {
log.error(e.getMessage(), e);
}
return tmp;
}
public static void printAll() {
if (INST == null) {
init();
}
log.info("[" + CONFIG_FILE + "] =============== start print config properties ===============");
if (INST != null && INST.prop != null) {
for (Map.Entry<Object, Object> entry : INST.prop.entrySet()) {
log.info("[" + CONFIG_FILE + "] " + entry.getKey() + "=" + entry.getValue());
}
}
log.info("[" + CONFIG_FILE + "] =============== end print config properties ===============");
}
public static String getString(String key) {
if (INST == null) {
init();
}
return INST.prop.getProperty(key);
}
public static String getString(String key, String defaultValue) {
String val = getString(key);
if (val == null) {
return defaultValue;
}
return val;
}
public static int getInt(String key, int defaultValue) {
String val = getString(key);
if (val != null) {
try {
return Integer.parseInt(val);
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
return defaultValue;
}
public static long getLong(String key, long defaultValue) {
String val = getString(key);
if (val != null) {
try {
return Long.parseLong(val);
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
return defaultValue;
}
}
suggest-task/src/main/java/com/secoo/so/suggest/db/DwDataSource.java 0 → 100644
package com.secoo.so.suggest.db;
import com.secoo.so.suggest.entity.SearchKeywordInfo;
import com.secoo.so.suggest.util.ObjectUtils;
import com.secoo.so.suggest.util.StringUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.dbcp.BasicDataSource;
import java.io.IOException;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.*;
@Slf4j
public class DwDataSource {
private DwDataSource() {
}
private static class DataSourceHolder {
private static BasicDataSource dataSource = new BasicDataSource();
static {
Properties prop = new Properties();
try {
prop.load(DwDataSource.class.getClassLoader().getResourceAsStream("db.properties"));
} catch (IOException e) {
log.error("init db config error", e);
}
dataSource.setDriverClassName("com.mysql.jdbc.Driver");
dataSource.setUrl(prop.getProperty("dw.read.url"));
dataSource.setUsername(prop.getProperty("dw.read.user"));
dataSource.setPassword(prop.getProperty("dw.read.password"));
dataSource.setTestWhileIdle(true);
dataSource.setTestOnReturn(true);
dataSource.setTimeBetweenEvictionRunsMillis(30000);
dataSource.setMaxActive(30);
dataSource.setTestOnBorrow(true);
dataSource.setValidationQuery("select 1 from dual");
}
}
public static Connection getConnection() {
try {
return DataSourceHolder.dataSource.getConnection();
} catch (SQLException e) {
log.error("获取链接异常", e);
throw new RuntimeException("获取链接异常:" + e.getMessage(), e);
}
}
public static void close() {
try {
DataSourceHolder.dataSource.close();
} catch (SQLException e) {
log.error("close error", e);
}
}
public static Map<String, Long> querySearchWordCountAndMaxId() {
Map<String, Long> result = new HashMap<>();
Connection conn = DwDataSource.getConnection();
PreparedStatement stmt = null;
ResultSet rs = null;
try {
String sql = "select count(*) as cnt, max(id) as max_id, min(id) as min_id from app_search_keyword_year_week_p_day";
stmt = conn.prepareStatement(sql);
rs = stmt.executeQuery();
while (rs.next()) {
Long count = rs.getLong("cnt");
Long maxId = rs.getLong("max_id");
Long minId = rs.getLong("min_id");
result.put("count", count);
result.put("maxId", maxId);
result.put("minId", minId);
}
} catch (Exception e) {
log.error("querySearchWordCountAndMaxId error", e);
} finally {
ObjectUtils.safeClose(conn, stmt, rs);
}
return result;
}
/**
* 查询品牌信息
*/
public static List<SearchKeywordInfo> querySearchKeywordInfoList(long startId, long endId) {
List<SearchKeywordInfo> searchKeywordInfoList = new ArrayList<>();
Connection conn = DwDataSource.getConnection();
PreparedStatement stmt = null;
ResultSet rs = null;
try {
String sql = "select id, keyword, year_pv, year_product_click_count, year_add_cart_count, "
+ " week_pv, week_product_click_count, week_add_cart_count, p_day, "
+ " week_uv, week_product_click_uv, week_add_cart_uv, "
+ " month_pv, month_product_click_count, month_add_cart_count, month_uv,"
+ " month_product_click_uv, month_add_cart_uv, prepare_tags "
+ " from app_search_keyword_year_week_p_day where id >= ? and id < ?";
stmt = conn.prepareStatement(sql);
stmt.setLong(1, startId);
stmt.setLong(2, endId);
rs = stmt.executeQuery();
while (rs.next()) {
Long id = rs.getLong("id");
String keyword = rs.getString("keyword");
if (StringUtils.isBlank(keyword)) {
continue;
}
String prepareTags = rs.getString("prepare_tags");
Integer yearPv = rs.getInt("year_pv");
Integer yearProductClickCount = rs.getInt("year_product_click_count");
Integer yearAddCartCount = rs.getInt("year_add_cart_count");
Long weekPv = rs.getLong("week_pv");
Long weekProductClickCount = rs.getLong("week_product_click_count");
Long weekAddCartCount = rs.getLong("week_add_cart_count");
Long weekUv = rs.getLong("week_uv");
Long weekProductClickUv = rs.getLong("week_product_click_uv");
Long weekAddCartUv = rs.getLong("week_add_cart_uv");
Long monthPv = rs.getLong("month_pv");
Long monthProductClickCount = rs.getLong("month_product_click_count");
Long monthAddCartCount = rs.getLong("month_add_cart_count");
Long monthUv = rs.getLong("month_uv");
Long monthProductClickUv = rs.getLong("month_product_click_uv");
Long monthAddCartUv = rs.getLong("month_add_cart_uv");
String pDay = rs.getString("p_day");
SearchKeywordInfo searchKeywordInfo = new SearchKeywordInfo();
searchKeywordInfo.setId(id);
searchKeywordInfo.setKeyword(keyword);
searchKeywordInfo.setPrepareTags(prepareTags);
searchKeywordInfo.setYearPv(yearPv);
searchKeywordInfo.setYearProductClickCount(yearProductClickCount);
searchKeywordInfo.setYearAddCartCount(yearAddCartCount);
searchKeywordInfo.setWeekPv(weekPv);
searchKeywordInfo.setWeekProductClickCount(weekProductClickCount);
searchKeywordInfo.setWeekAddCartCount(weekAddCartCount);
searchKeywordInfo.setWeekUv(weekUv);
searchKeywordInfo.setWeekProductClickUv(weekProductClickUv);
searchKeywordInfo.setWeekAddCartUv(weekAddCartUv);
searchKeywordInfo.setMonthPv(monthPv);
searchKeywordInfo.setMonthProductClickCount(monthProductClickCount);
searchKeywordInfo.setMonthAddCartCount(monthAddCartCount);
searchKeywordInfo.setMonthUv(monthUv);
searchKeywordInfo.setMonthProductClickUv(monthProductClickUv);
searchKeywordInfo.setMonthAddCartUv(monthAddCartUv);
searchKeywordInfo.setPDay(pDay);
searchKeywordInfoList.add(searchKeywordInfo);
}
} catch (Exception e) {
log.error("querySearchKeywordInfoList error", e);
} finally {
ObjectUtils.safeClose(conn, stmt, rs);
}
return searchKeywordInfoList;
}
}
suggest-task/src/main/java/com/secoo/so/suggest/db/ErpDataSource.java 0 → 100644
package com.secoo.so.suggest.db;
import com.secoo.so.suggest.entity.BrandInfo;
import com.secoo.so.suggest.entity.CategoryInfo;
import com.secoo.so.suggest.util.ObjectUtils;
import com.secoo.so.suggest.util.StringUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.dbcp.BasicDataSource;
import java.io.IOException;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
@Slf4j
public class ErpDataSource {
private ErpDataSource() {
}
private static class DataSourceHolder {
private static BasicDataSource dataSource = new BasicDataSource();
static {
Properties prop = new Properties();
try {
prop.load(ErpDataSource.class.getClassLoader().getResourceAsStream("db.properties"));
} catch (IOException e) {
log.error("init config error", e);
}
dataSource.setDriverClassName("com.mysql.jdbc.Driver");
dataSource.setUrl(prop.getProperty("erp.read.url"));
dataSource.setUsername(prop.getProperty("erp.read.user"));
dataSource.setPassword(prop.getProperty("erp.read.password"));
dataSource.setTestWhileIdle(true);
dataSource.setTestOnReturn(true);
dataSource.setTimeBetweenEvictionRunsMillis(30000);
dataSource.setMaxActive(30);
dataSource.setTestOnBorrow(true);
dataSource.setValidationQuery("select 1 from dual");
}
}
public static Connection getConnection() {
try {
return DataSourceHolder.dataSource.getConnection();
} catch (SQLException e) {
log.error("获取链接异常", e);
throw new RuntimeException("获取链接异常:" + e.getMessage(), e);
}
}
public static void close() {
try {
DataSourceHolder.dataSource.close();
} catch (SQLException e) {
log.error("close error", e);
}
}
/**
* 查询品牌信息
*/
public static List<BrandInfo> queryBrandInfoList() {
List<BrandInfo> brandInfoList = new ArrayList<>();
Connection conn = ErpDataSource.getConnection();
PreparedStatement stmt = null;
ResultSet rs = null;
try {
String sql = "select id,en_name,ch_name,short_name,nickname from secooErpDB.t_product_brand where is_del = 0 and enabled = 1";
stmt = conn.prepareStatement(sql);
rs = stmt.executeQuery();
while (rs.next()) {
Long id = rs.getLong("id");
if (id == null || id <= 0) {
continue;
}
String enName = rs.getString("en_name");
String chName = rs.getString("ch_name");
String shortName = rs.getString("short_name");
String nickName = rs.getString("nickname");
BrandInfo brandInfo = new BrandInfo();
brandInfo.setId(id);
brandInfo.setEnName(enName);
brandInfo.setChName(chName);
brandInfo.setShortName(shortName);
brandInfo.setNickName(nickName);
brandInfoList.add(brandInfo);
}
} catch (Exception e) {
log.error("queryBrandInfoList error", e);
} finally {
ObjectUtils.safeClose(conn, stmt, rs);
}
return brandInfoList;
}
/**
* 查询品牌信息
*/
public static List<CategoryInfo> queryCategoryInfoList() {
List<CategoryInfo> categoryInfoList = new ArrayList<>();
Connection conn = ErpDataSource.getConnection();
PreparedStatement stmt = null;
ResultSet rs = null;
try {
String sql = "select id,name from secooErpDB.t_product_category where is_del = 0 and enabled = 1";
stmt = conn.prepareStatement(sql);
rs = stmt.executeQuery();
while (rs.next()) {
Long id = rs.getLong("id");
String name = rs.getString("name");
if (id == null || id <= 0 || StringUtils.isBlank(name)) {
continue;
}
CategoryInfo categoryInfo = new CategoryInfo();
categoryInfo.setId(id);
categoryInfo.setName(name);
categoryInfoList.add(categoryInfo);
}
} catch (Exception e) {
log.error("queryCategoryInfoList error", e);
} finally {
ObjectUtils.safeClose(conn, stmt, rs);
}
return categoryInfoList;
}
}
suggest-task/src/main/java/com/secoo/so/suggest/db/SeoDataSource.java 0 → 100644
package com.secoo.so.suggest.db;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.dbcp.BasicDataSource;
import java.io.IOException;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.Properties;
@Slf4j
public class SeoDataSource {
private SeoDataSource() {
}
private static class DataSourceHolder {
private static BasicDataSource dataSource = new BasicDataSource();
static {
Properties prop = new Properties();
try {
prop.load(SeoDataSource.class.getClassLoader().getResourceAsStream("db.properties"));
} catch (IOException e) {
log.error("init config error", e);
}
dataSource.setDriverClassName("com.mysql.jdbc.Driver");
dataSource.setUrl(prop.getProperty("seo.read.url"));
dataSource.setUsername(prop.getProperty("seo.read.user"));
dataSource.setPassword(prop.getProperty("seo.read.password"));
dataSource.setTestWhileIdle(true);
dataSource.setTestOnReturn(true);
dataSource.setTimeBetweenEvictionRunsMillis(30000);
dataSource.setMaxActive(30);
dataSource.setTestOnBorrow(true);
dataSource.setValidationQuery("select 1 from dual");
}
}
public static Connection getConnection() {
try {
return DataSourceHolder.dataSource.getConnection();
} catch (SQLException e) {
log.error("获取链接异常", e);
throw new RuntimeException("获取链接异常:" + e.getMessage(), e);
}
}
public static void close() {
try {
DataSourceHolder.dataSource.close();
} catch (SQLException e) {
log.error("close error", e);
}
}
}
suggest-task/src/main/java/com/secoo/so/suggest/entity/BrandInfo.java 0 → 100644
package com.secoo.so.suggest.entity;
import lombok.Data;
import java.io.Serializable;
/**
* 品牌信息
*/
@Data
public class BrandInfo implements Serializable {
private static final long serialVersionUID = -6388347520294644169L;
private Long id;
private String enName;
private String chName;
private String shortName;
private String nickName;
}
suggest-task/src/main/java/com/secoo/so/suggest/entity/CategoryInfo.java 0 → 100644
package com.secoo.so.suggest.entity;
import lombok.Data;
import java.io.Serializable;
/**
* 品类信息
*/
@Data
public class CategoryInfo implements Serializable {
private static final long serialVersionUID = -12528308204568143L;
private Long id;
private String name;
}
suggest-task/src/main/java/com/secoo/so/suggest/entity/EsSuggestKeywordInfo.java 0 → 100644
package com.secoo.so.suggest.entity;
import lombok.Data;
import java.io.Serializable;
@Data
public class EsSuggestKeywordInfo implements Serializable {
private static final long serialVersionUID = -2891215162084524117L;
private String keyword;
private String keywordPinYin;
private Integer yearCount;
private Integer yearClickCount;
private Integer yearCartCount;
private Long weekCount;
private Long weekClickCount;
private Long weekCartCount;
private Double yearClickRatio;
private Double yearCartRatio;
private Double weekClickRatio;
private Double weekCartRatio;
private Boolean isBrand;
private Boolean isCategory;
private Boolean isManual;
private Boolean isSensitive;
private Integer manualValue;
private Double wordRank;
private Double wordABRank;
private String keywordVersion;
private Boolean isEuropeWord;
private String suggestTags;
private Long updateTime;
}
suggest-task/src/main/java/com/secoo/so/suggest/entity/SearchKeywordInfo.java 0 → 100644
package com.secoo.so.suggest.entity;
import lombok.Data;
import java.io.Serializable;
/**
* 搜索词信息
* app_search_keyword_year_week_p_day
*/
@Data
public class SearchKeywordInfo implements Serializable {
private static final long serialVersionUID = 5479160854636000122L;
private Long id;
private String keyword;
private String prepareTags;
private Integer yearPv;
private Integer yearProductClickCount;
private Integer yearAddCartCount;
private Long weekPv;
private Long weekProductClickCount;
private Long weekAddCartCount;
private Long weekUv;
private Long weekProductClickUv;
private Long weekAddCartUv;
private Long monthPv;
private Long monthProductClickCount;
private Long monthAddCartCount;
private Long monthUv;
private Long monthProductClickUv;
private Long monthAddCartUv;
private String pDay;
}
suggest-task/src/main/java/com/secoo/so/suggest/es/ESException.java 0 → 100644
package com.secoo.so.suggest.es;
public class ESException extends Exception {
private static final long serialVersionUID = -4947060289056203488L;
private String msg;
private int code = 500;
public ESException(String msg) {
super(msg);
this.msg = msg;
}
public ESException(String msg, Throwable e) {
super(msg, e);
this.msg = msg;
}
public ESException(String msg, int code) {
super(msg);
this.msg = msg;
this.code = code;
}
public ESException(String msg, int code, Throwable e) {
super(msg, e);
this.msg = msg;
this.code = code;
}
public String getMsg() {
return msg;
}
public void setMsg(String msg) {
this.msg = msg;
}
public int getCode() {
return code;
}
public void setCode(int code) {
this.code = code;
}
}
suggest-task/src/main/java/com/secoo/so/suggest/es/EsClient.java 0 → 100644
package com.secoo.so.suggest.es;
import com.alibaba.fastjson.JSON;
import com.secoo.so.suggest.util.CollectionUtils;
import com.secoo.so.suggest.util.ObjectUtils;
import com.secoo.so.suggest.util.StringUtils;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.entity.ContentType;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
import org.apache.http.nio.entity.NStringEntity;
import org.apache.http.util.EntityUtils;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.*;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.client.*;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.rest.RestRequest;
import org.elasticsearch.search.Scroll;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.sort.SortBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.*;
public class EsClient {
private static final Logger logger = LoggerFactory.getLogger(EsClient.class);
public static final String SCHEMA_HTTP = "http"; // 默认http
public static final String SCHEMA_HTTPS = "https";
/**
* host地址作为key，缓存client
*/
private static final Map<String, EsClient> clientMap = new HashMap<>();
private static boolean uniqueConnectTimeConfig = false;
private static boolean uniqueConnectNumConfig = false;
private static int CONNECT_TIME_OUT = 1000;
private static int SOCKET_TIME_OUT = 30000;
private static int CONNECTION_REQUEST_TIME_OUT = 500;
private static int MAX_CONNECT_NUM = 100;
private static int MAX_CONNECT_PER_ROUTE = 100;
private String schema; // http | https
private String host; // 主机地址，如果是集群： 主机名加分号分隔
private int port; // 端口号
private String authUser;
private String authPassword;
private RestClientBuilder builder;
private RestClient restClient;
private RestHighLevelClient restHighLevelClient;
private EsClient(String url) {
this(url, null, null);
}
private EsClient(String url, String username, String password) {
if (StringUtils.isBlank(url)) {
throw new IllegalArgumentException("url may not be blank");
}
String hostText = url;
String schema = null;
int schemeIdx = url.indexOf("://");
if (schemeIdx > 0) {
schema = url.substring(0, schemeIdx);
hostText = url.substring(schemeIdx + 3);
}
int port = 80;
int portIdx = hostText.lastIndexOf(":");
if (portIdx > 0) {
try {
port = Integer.parseInt(hostText.substring(portIdx + 1));
} catch (NumberFormatException var7) {
throw new IllegalArgumentException("Invalid HTTP host: " + hostText);
}
hostText = hostText.substring(0, portIdx);
}
this.init(schema, hostText, port, username, password);
}
private EsClient(String schema, String host, int port) {
this(schema, host, port, null, null);
}
private EsClient(String schema, String host, int port, String username, String password) {
this.init(schema, host, port, username, password);
}
private void init(String schema, String host, int port, String username, String password) {
this.schema = StringUtils.isNotBlank(schema) ? schema : SCHEMA_HTTP;
this.host = host;
this.port = port;
this.authUser = username;
this.authPassword = password;
String hostList = this.host;
String[] split = hostList.split(";");
HttpHost[] hostArray = new HttpHost[split.length];
for (int i = 0; i < split.length; i++) {
hostArray[i] = new HttpHost(split[i], this.port, this.schema);
}
// 设置超时时间1小时
this.builder = RestClient.builder(hostArray).setMaxRetryTimeoutMillis(60 * 60 * 1000);
if (uniqueConnectTimeConfig) {
setConnectTimeOutConfig(this.builder);
}
if (uniqueConnectNumConfig) {
setMultiConnectConfig(this.builder);
}
// 设置账户密码
if (StringUtils.isNotBlank(this.authUser, this.authPassword)) {
setDefaultCredentialsProvider(this.builder, this.authUser, this.authPassword);
}
this.restClient = builder.build();
this.restHighLevelClient = new RestHighLevelClient(builder);
logger.info("build es client success: schema = {}, host = {}, port = {}", this.schema, this.host, this.port);
}
private static String buildEsClientUniqueId(String schema, String host, int port, String authUser, String authPassword) {
return (StringUtils.isNotBlank(schema) ? schema : SCHEMA_HTTP) + "|" + host + "|" + port + "|" + authUser + "|" + authPassword;
}
private String getEsClientUniqueId() {
return buildEsClientUniqueId(this.schema, this.host, this.port, this.authUser, this.authPassword);
}
public static synchronized EsClient getEsClient(String schema, String host, int port) {
return getEsClient(schema, schema, port, null, null);
}
public static synchronized EsClient getEsClient(String schema, String host, int port, String authUser, String authPassword) {
String uniqueId = buildEsClientUniqueId(schema, host, port, authUser, authPassword);
if (!clientMap.containsKey(uniqueId)) {
clientMap.put(uniqueId, buildEsClient(schema, host, port));
}
return clientMap.get(uniqueId);
}
public static EsClient buildEsClient(String url) {
return new EsClient(url);
}
public static EsClient buildEsClient(String url, String user, String password) {
return new EsClient(url, user, password);
}
public static EsClient buildEsClient(String schema, String host, int port) {
return new EsClient(schema, host, port);
}
public static EsClient buildEsClient(String schema, String host, int port, String user, String password) {
return new EsClient(schema, host, port, user, password);
}
/**
* 配置账号密码
*/
private static void setDefaultCredentialsProvider(RestClientBuilder builder, String authUser, String authPassword) {
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(authUser, authPassword));
builder.setHttpClientConfigCallback(httpClientBuilder -> {
httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
return httpClientBuilder;
});
}
/**
* 主要关于异步httpclient的连接延时配置
*/
private static void setConnectTimeOutConfig(RestClientBuilder builder) {
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
@Override
public RequestConfig.Builder customizeRequestConfig(
RequestConfig.Builder builder) {
builder.setConnectTimeout(CONNECT_TIME_OUT);
builder.setSocketTimeout(SOCKET_TIME_OUT);
builder.setConnectionRequestTimeout(CONNECTION_REQUEST_TIME_OUT);
return builder;
}
});
}
/**
* 主要关于异步httpclient的连接数配置
*/
private static void setMultiConnectConfig(RestClientBuilder builder) {
builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
@Override
public HttpAsyncClientBuilder customizeHttpClient(
HttpAsyncClientBuilder httpAsyncClientBuilder) {
httpAsyncClientBuilder.setMaxConnTotal(MAX_CONNECT_NUM);
httpAsyncClientBuilder
.setMaxConnPerRoute(MAX_CONNECT_PER_ROUTE);
return httpAsyncClientBuilder;
}
});
}
public void close() {
EsClient.close(this);
}
/**
* 关闭资源
*/
public static void close(EsClient esClient) {
if (esClient != null) {
try {
String uniqueId = esClient.getEsClientUniqueId();
if (clientMap.containsKey(esClient)) {
clientMap.remove(uniqueId);
}
if (esClient.restHighLevelClient != null) {
esClient.restHighLevelClient.close();
}
if (esClient.restClient != null) {
esClient.restClient.close();
}
} catch (Exception e) {
logger.error("close esClient error.", e);
}
}
}
/**
* 单条插入(使用es自动生成的id)
*
* @param indexName
* @param object
*/
public void addWithoutId(String indexName, Object object) {
IndexRequest indexRequest = new IndexRequest(indexName, indexName);
indexRequest.source(JSON.toJSONString(object), XContentType.JSON);
try {
restHighLevelClient.index(indexRequest, RequestOptions.DEFAULT);
IndexResponse response = restHighLevelClient.index(indexRequest, RequestOptions.DEFAULT);
} catch (IOException e) {
logger.error("insert object error, index={}, object={}", indexName, JSON.toJSONString(object), e);
} catch (RuntimeException e) {
logger.error("insert object error, index={}, object={}", indexName, JSON.toJSONString(object), e);
throw e;
}
}
public void add(String indexName, EsObject esObject) {
if (esObject.getObject() == null) {
return;
}
String objectJson = null;
if (esObject.getObject() instanceof String) {
objectJson = (String) esObject.getObject();
// TODO： check is json
} else {
objectJson = JSON.toJSONString(esObject.getObject());
}
//这里原本是非的关系，但为了支持id为null的情况，改为且
if (esObject.getId() != null && esObject.getId().getBytes().length >= 512) {
logger.warn("es insert error, id is too long, must be no longer than 512 bytes. esObject : {}", JSON.toJSONString(esObject));
return;
}
try {
switch (esObject.getOpType()) {
case UPDATE:
UpdateRequest updateRequest = new UpdateRequest(indexName, indexName,
esObject.getId());
updateRequest.doc(objectJson, XContentType.JSON);
UpdateResponse updateResponse = restHighLevelClient.update(updateRequest, RequestOptions.DEFAULT);
break;
case CREATE:
case INDEX:
IndexRequest indexRequest = new IndexRequest(indexName, indexName);
indexRequest.opType(esObject.getOpType());
if (StringUtils.isNotBlank(esObject.getId())) {
indexRequest.id(esObject.getId());
}
indexRequest.source(objectJson, XContentType.JSON);
IndexResponse indexResponse = restHighLevelClient.index(indexRequest, RequestOptions.DEFAULT);
break;
}
} catch (IOException e) {
logger.error("add<{}> object error, index={}, object= " + JSON.toJSONString(esObject), esObject.getOpType().getLowercase(), indexName, e);
} catch (RuntimeException e) {
logger.error("add<{}> object error, index={}, object= " + JSON.toJSONString(esObject), esObject.getOpType().getLowercase(), indexName, e);
throw e;
}
}
public void batchDelete(String indexName, String typeName, List<String> idList) {
if (CollectionUtils.isNotEmpty(idList)) {
List<EsObject> objList = new ArrayList<>();
for (String id : idList) {
objList.add(new EsObject(id, null, EsObject.DELETE));
}
this.safeBatch(indexName, typeName, objList);
}
}
public void safeBatch(String indexName, List<EsObject> objectList) {
safeBatch(indexName, indexName, objectList);
}
public void safeBatch(String indexName, String typeName, List<EsObject> objectList) {
try {
batch(indexName, typeName, objectList);
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
/**
* 支持重试的批量处理，重试次数自己指定
*
* @param indexName
* @param objectList
* @param retryCount
*/
public void batchSupportRetry(String indexName, List<EsObject> objectList, Integer retryCount) {
try {
batch(indexName, indexName, objectList);
} catch (IOException | RuntimeException e) {
logger.error(e.getMessage(), e);
} catch (ESException ese) {
logger.error(ese.getMsg(), ese);
if (retryCount == 0) {
logger.error("Batch execute index over, cause run out of retries!");
} else {
retryCount--;
logger.info("Batch execute index retry begin by count<{}>", retryCount);
batchSupportRetry(indexName, objectList, retryCount);
}
}
}
/**
* 批量插入
* <p>
* bulk 可以执行的操作类型：
* <p>
* （1）delete：删除一个文档，只要1个json串就可以了
* （2）create：PUT /index/type/id/_create，强制创建
* （3）index：普通的put操作，可以是创建文档，也可以是全量替换文档
* （4）update：执行的partial update操作
*
* @param indexName
* @param typeName
* @param objectList
*/
public void batch(String indexName, String typeName, List<EsObject> objectList) throws RuntimeException, IOException, ESException {
if (objectList != null) {
BulkRequest bulkRequest = new BulkRequest();
List<String> idList = new ArrayList<>();
int count = 1;
int size = objectList.size();
for (EsObject esObject : objectList) {
// 去掉原有的id为空的判断
if (esObject.getObject() == null && !EsObject.DELETE.equals(esObject.getOpType())) {
continue;
}
String objectJson = null;
if (esObject.getObject() instanceof String) {
objectJson = (String) esObject.getObject();
// TODO： check is json
} else {
objectJson = JSON.toJSONString(esObject.getObject());
}
//这里原本是非的关系，但为了支持id为null的情况，改为且
if (esObject.getId() != null && esObject.getId().getBytes().length >= 512) {
logger.warn("es insert error, id is too long, must be no longer than 512 bytes. esObject : {}", JSON.toJSONString(esObject));
continue;
}
switch (esObject.getOpType()) {
case DELETE:
DeleteRequest deleteRequest = new DeleteRequest(indexName, typeName, esObject.getId());
bulkRequest.add(deleteRequest);
break;
case UPDATE:
UpdateRequest updateRequest = new UpdateRequest(indexName, typeName, esObject.getId());
updateRequest.docAsUpsert(true);
updateRequest.doc(objectJson, XContentType.JSON);
bulkRequest.add(updateRequest);
break;
case CREATE:
case INDEX:
default:
IndexRequest indexRequest = new IndexRequest(indexName, typeName);
if (esObject.getOpType() != null) {
indexRequest.opType(esObject.getOpType());
}
if (StringUtils.isNotBlank(esObject.getId())) {
indexRequest.id(esObject.getId());
}
indexRequest.source(objectJson, XContentType.JSON);
bulkRequest.add(indexRequest);
break;
}
idList.add(esObject.getId());
// 每5000条批量插入一次
if (count % 5000 == 0 || count == size) {
long start = System.currentTimeMillis();
try {
BulkResponse response = restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT);
if (response.hasFailures()) {
//取第一个返回错误的抛出
Iterator<BulkItemResponse> responseIt = response.iterator();
while (responseIt.hasNext()) {
BulkItemResponse responseItem = responseIt.next();
if (responseItem.isFailed()) {
logger.error("batch insert to index<{}> error: {}, cost = {}ms", responseItem.getIndex(), responseItem.getFailureMessage(), (System.currentTimeMillis() - start));
// throw new ESException(responseItem.getFailureMessage(), responseItem.getFailure().getStatus().getStatus());
}
}
} else {
logger.info("batch insert to index<{}> process ( {} / {} ) response status: {}, cost = {}ms", indexName, count, size, response.status(), (System.currentTimeMillis() - start));
}
} catch (IOException e) {
logger.error("batch insert error idList: {}", JSON.toJSONString(idList));
logger.error("batch insert error, index={}, type={}, cost = {}ms", indexName, typeName, (System.currentTimeMillis() - start), e);
throw e;
} catch (RuntimeException e) {
logger.error("batch insert error idList: {}", JSON.toJSONString(idList));
logger.error("batch insert error, index={}, type={}, cost = {}ms", indexName, typeName, (System.currentTimeMillis() - start), e);
throw e;
}
bulkRequest = new BulkRequest();
idList = new ArrayList<>();
}
count++;
}
}
}
/**
* 单条删除
*
* @param indexName
* @param id
*/
public void delete(String indexName, String id) {
DeleteRequest deleteRequest = new DeleteRequest(indexName, indexName, id);
try {
DeleteResponse response = restHighLevelClient.delete(deleteRequest, RequestOptions.DEFAULT);
} catch (IOException e) {
logger.error("delete id<{}> from {} error.", id, indexName, e);
}
}
/**
* 单条删除
*
* @param indexName
* @param id
*/
public void delete(String indexName, String indexType, String id) {
DeleteRequest deleteRequest = new DeleteRequest(indexName, indexType, id);
try {
DeleteResponse response = restHighLevelClient.delete(deleteRequest, RequestOptions.DEFAULT);
} catch (IOException e) {
logger.error("delete id<{}> from {} - {} error.", id, indexName, indexType, e);
}
}
public void deleteByQuery(String indexName, QueryBuilder queryBuilder) {
deleteByQuery(indexName, queryBuilder, 0);
}
/**
* 根据查询删除数据
*
* @param indexName
* @param queryBuilder
* @param scroll
*/
public void deleteByQuery(String indexName, QueryBuilder queryBuilder, int scroll) {
String requestBody = null;
try {
if (null == queryBuilder) {
return;
}
requestBody = queryBuilder.toString();
if (StringUtils.isBlank(requestBody)) {
return;
}
requestBody = "{\"query\":" + requestBody + "}";
String endpoint = "/" + indexName + "/_delete_by_query?conflicts=proceed";
if (scroll != 0) {
endpoint += "&scroll_size=" + scroll;
}
Request request = new Request(RestRequest.Method.POST.toString(), endpoint);
request.setEntity(new NStringEntity(requestBody, ContentType.APPLICATION_JSON));
Response response = restHighLevelClient.getLowLevelClient().performRequest(request);
logger.info("delete success, response {} ", EntityUtils.toString(response.getEntity()));
} catch (Exception e) {
logger.error("delete index<{}> from {} {}error.", indexName, requestBody, e.getMessage());
}
}
/**
* 查询是否存在field=value的document
*/
public Boolean checkExist(String indexName, String field, Object... value) {
if (StringUtils.isNotBlank(indexName) && StringUtils.isNotBlank(field) && value.length > 0) {
try {
return searchCount(indexName, QueryBuilders.termsQuery(field, value)) > 0;
} catch (Exception e) {
logger.error("checkExist field: {} = {} in {} error.", field, value, indexName, e);
}
}
return false;
}
/**
* 根据字段值列表查询（in）
*
* @author xupeng
* @date: 2019-01-02
*/
public List<Map<String, Object>> findByValueList(String indexName, String field, Object value) {
List<Object> valueList = new ArrayList<>();
if (value != null) {
valueList.add(value);
}
return findByValueList(indexName, field, valueList);
}
/**
* 不支持回调的查询方法
*
* @param indexName
* @param queryBuilder
* @return
*/
public List<Map<String, Object>> search(String indexName, QueryBuilder queryBuilder) {
return search(indexName, queryBuilder, null, null, 5000, null);
}
/**
* 直接返回具体对象
*
* @param indexName
* @param queryBuilder
* @param clazz
* @param <T>
* @return
*/
public <T> List<T> search(String indexName, QueryBuilder queryBuilder, Class<T> clazz) {
return ObjectUtils.listToObjects(search(indexName, queryBuilder), clazz);
}
public List<Map<String, Object>> search(String indexName, QueryBuilder queryBuilder, EsSearchCallback callback) {
return search(indexName, queryBuilder, null, callback, 5000, null);
}
public <T> List<T> search(String indexName, QueryBuilder queryBuilder, EsSearchCallback callback, Class<T> clazz) {
return ObjectUtils.listToObjects(search(indexName, queryBuilder, callback), clazz);
}
public <T> List<T> search(String indexName, QueryBuilder queryBuilder, EsSearchCallback callback, int limit, Class<T> clazz) {
return ObjectUtils.listToObjects(search(indexName, queryBuilder, null, callback, limit, null), clazz);
}
public List<Map<String, Object>> search(String indexName, QueryBuilder queryBuilder, EsSearchCallback callback, int limit) {
return search(indexName, queryBuilder, null, callback, limit, null);
}
public List<Map<String, Object>> search(String indexName, QueryBuilder queryBuilder, String[] storeFields) {
return search(indexName, queryBuilder, storeFields, null, 5000, null);
}
public <T> List<T> search(String indexName, QueryBuilder queryBuilder, String[] storeFields, Class<T> clazz) {
return ObjectUtils.listToObjects(search(indexName, queryBuilder, storeFields, null, 5000, null), clazz);
}
public <T> List<T> search(String indexName, QueryBuilder queryBuilder, String[] storeFields, SortBuilder[] sortBuilders, Class<T> clazz) {
return ObjectUtils.listToObjects(search(indexName, queryBuilder, storeFields, null, 5000, sortBuilders), clazz);
}
/**
* 根据QueryBuilder查询，直接回调函数中直接处理数据
*
* @author xupeng
* @date: 2019-01-25
*/
public List<Map<String, Object>> search(String indexName, QueryBuilder queryBuilder, String[] storeFields, EsSearchCallback callback, int limit, SortBuilder[] sortBuilders) {
List<Map<String, Object>> resultList = new ArrayList<>();
List<String> scrollIdList = new ArrayList<>();
if (StringUtils.isNotBlank(indexName) && queryBuilder != null) {
try {
SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.searchSource().query(queryBuilder);
if (null != storeFields && storeFields.length != 0) {
searchSourceBuilder.fetchSource(storeFields, null);
}
if (null != sortBuilders && sortBuilders.length != 0) {
for (SortBuilder sortBuilder : sortBuilders) {
searchSourceBuilder.sort(sortBuilder);
}
}
searchSourceBuilder.size(limit);
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.searchType(SearchType.QUERY_THEN_FETCH);
searchRequest.source(searchSourceBuilder);
Scroll scroll = new Scroll(TimeValue.timeValueSeconds(60));
searchRequest.scroll(scroll);
SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
long count = 0;
while (searchResponse != null) {
// 记录scrollId
String scrollId = searchResponse.getScrollId();
if (StringUtils.isNotBlank(scrollId) && !scrollIdList.contains(scrollId)) {
scrollIdList.add(scrollId);
}
// 获取上次查询的结果
if (searchResponse.getHits().getHits().length > 0) {
count = count + searchResponse.getHits().getHits().length;
searchResponse.getHits().forEach(documentFields -> {
Map map = documentFields.getSourceAsMap();
map.put("_id", documentFields.getId());
map.put("es_id", documentFields.getId());
resultList.add(documentFields.getSourceAsMap());
});
if (null != callback) {
try {
callback.callback(resultList);
//清空resultList
resultList.clear();
} catch (Exception e) {
logger.error("search index<{}> EsSearchCallback process error : {}", indexName, e.getMessage());
}
}
// 执行下一次查询
if (StringUtils.isNotBlank(scrollId)
&& searchResponse.getHits().getTotalHits() > count) {
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
scrollRequest.scroll(scroll);
searchResponse = restHighLevelClient.scroll(scrollRequest, RequestOptions.DEFAULT);
} else {
break;
}
} else {
// 没有结果结束查询
break;
}
}
} catch (Exception e) {
logger.error("search index<{}> error, QueryBuilder = {} .", indexName, JSON.toJSONString(queryBuilder), e);
} finally {
clearScroll(scrollIdList);
}
}
return resultList;
}
/**
* 根据QueryBuilder查询，查询limit限制的数据条数，不支持排序
*
* @param indexName
* @param queryBuilder
* @param limit
* @return
*/
public List<Map<String, Object>> search(String indexName, QueryBuilder queryBuilder, int limit) {
return search(indexName, queryBuilder, limit, null);
}
public <T> List<T> search(Class<T> clazz, String indexName, QueryBuilder queryBuilder, int limit) {
return ObjectUtils.listToObjects(search(indexName, queryBuilder, limit), clazz);
}
public <T> List<T> search(Class<T> clazz, String indexName, QueryBuilder queryBuilder, int limit, SortBuilder... sortBuilders) {
return ObjectUtils.listToObjects(search(indexName, queryBuilder, limit, sortBuilders), clazz);
}
/**
* 根据QueryBuilder查询，查询limit限制的数据条数，支持排序
*
* @param indexName
* @param queryBuilder
* @param limit
* @param sortBuilders
* @return
*/
public List<Map<String, Object>> search(String indexName, QueryBuilder queryBuilder, int limit, SortBuilder... sortBuilders) {
List<Map<String, Object>> resultList = new ArrayList<>();
if (StringUtils.isNotBlank(indexName) && queryBuilder != null) {
SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.searchSource().query(queryBuilder);
searchSourceBuilder.size(limit);
if (null != sortBuilders && sortBuilders.length != 0) {
for (SortBuilder sortBuilder : sortBuilders) {
searchSourceBuilder.sort(sortBuilder);
}
}
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.searchType(SearchType.QUERY_THEN_FETCH);
searchRequest.source(searchSourceBuilder);
try {
SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
if (searchResponse == null || searchResponse.getHits().getHits().length < 0) {
return resultList;
}
searchResponse.getHits().forEach(documentFields -> {
// 直接使用source返回的map会丢失es_id
Map<String, Object> sourceAsMap = documentFields.getSourceAsMap();
sourceAsMap.put("_id", documentFields.getId());
sourceAsMap.put("es_id", documentFields.getId());
resultList.add(sourceAsMap);
});
} catch (IOException e) {
logger.error("search index<{}> error, QueryBuilder = {} .", indexName, JSON.toJSONString(queryBuilder), e);
}
return resultList;
}
return resultList;
}
public Map<String, Object> getOne(String indexName, QueryBuilder queryBuilder, SortBuilder... sortBuilders) {
List<Map<String, Object>> list = search(indexName, queryBuilder, 1, sortBuilders);
if (CollectionUtils.isNotEmpty(list)) {
return list.get(0);
}
return null;
}
public <T> T getOne(Class<T> clazz, String indexName, QueryBuilder queryBuilder, SortBuilder... sortBuilders) {
try {
return ObjectUtils.mapToObjectWithJSON(getOne(indexName, queryBuilder, sortBuilders), clazz);
} catch (Exception e) {
logger.error(e.getMessage(), e);
return null;
}
}
public Map<String, Object> getById(String indexName, String id) {
return getOne(indexName, QueryBuilders.idsQuery().addIds(id));
}
public <T> T getById(String indexName, String id, Class<T> clazz) {
try {
return ObjectUtils.mapToObject(getById(indexName, id), clazz);
} catch (Exception e) {
logger.error(e.getMessage(), e);
return null;
}
}
/**
* 根据QueryBuilder查询
*
* @author xupeng
* @date: 2019-01-25
*/
public long searchCount(String indexName, QueryBuilder queryBuilder) {
if (StringUtils.isNotBlank(indexName) && queryBuilder != null) {
try {
SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.searchSource().query(queryBuilder);
searchSourceBuilder.size(0);
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.searchType(SearchType.QUERY_THEN_FETCH);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
if (searchResponse != null) {
return searchResponse.getHits().getTotalHits();
}
} catch (Exception e) {
logger.error("search index<{}> count error, QueryBuilder = {} .", indexName, JSON.toJSONString(queryBuilder), e);
}
}
return 0;
}
/**
* 根据字段值列表查询（in）
*
* @author xupeng
* @date: 2019-01-25
*/
public List<Map<String, Object>> findByValueList(String indexName, String field, List<Object> valueList) {
if (StringUtils.isNotBlank(indexName) && StringUtils.isNotBlank(field)
&& valueList != null && !valueList.isEmpty()) {
// 剔除value里的null
List<Object> fixValueList = new ArrayList<>();
valueList.forEach(value -> {
if (value != null && (!(value instanceof String) || ((String) value).trim().length() > 0)) {
fixValueList.add(value);
}
});
if (fixValueList != null && !fixValueList.isEmpty()) {
QueryBuilder queryBuilder = QueryBuilders.termsQuery(field, fixValueList);
return search(indexName, queryBuilder);
}
}
return null;
}
public <T> List<T> findListByValueList(String indexName, String field, List<Object> valueList, Class<T> clazz) {
List<T> resultList = new ArrayList<>();
List<Map<String, Object>> mapList = findByValueList(indexName, field, valueList);
if (CollectionUtils.isNotEmpty(mapList)) {
mapList.forEach(map -> {
resultList.add(JSON.parseObject(JSON.toJSONString(map), clazz));
});
}
return resultList;
}
/**
* 清除滚动ID
*/
public boolean clearScroll(List<String> scrollIdList) {
if (scrollIdList != null && !scrollIdList.isEmpty()) {
try {
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
clearScrollRequest.scrollIds(scrollIdList);
ClearScrollResponse clearScrollResponse = restHighLevelClient.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
return clearScrollResponse.isSucceeded();
} catch (Exception e) {
logger.error("clearScroll error: {} ", JSON.toJSONString(scrollIdList), e);
}
}
return true;
}
/**
* 清除滚动ID
*/
public boolean clearScroll(String scrollId) {
if (StringUtils.isNotBlank(scrollId)) {
try {
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
clearScrollRequest.addScrollId(scrollId);
ClearScrollResponse clearScrollResponse = restHighLevelClient.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
return clearScrollResponse.isSucceeded();
} catch (Exception e) {
logger.error("clearScroll error: {} ", scrollId, e);
}
}
return true;
}
}
suggest-task/src/main/java/com/secoo/so/suggest/es/EsObject.java 0 → 100644
package com.secoo.so.suggest.es;
import org.elasticsearch.action.DocWriteRequest;
import java.io.Serializable;
public class EsObject implements Serializable {
private static final long serialVersionUID = -3593470306368703625L;
public static final DocWriteRequest.OpType INDEX = DocWriteRequest.OpType.INDEX;
public static final DocWriteRequest.OpType CREATE = DocWriteRequest.OpType.CREATE;
public static final DocWriteRequest.OpType UPDATE = DocWriteRequest.OpType.UPDATE;
public static final DocWriteRequest.OpType DELETE = DocWriteRequest.OpType.DELETE;
private String id;
private Object object;
private DocWriteRequest.OpType opType = DocWriteRequest.OpType.INDEX;
public EsObject() {
}
public EsObject(String id, Object object) {
this.id = id;
this.object = object;
}
/**
* DocWriteRequest.OpType.INDEX : 覆盖更新
* DocWriteRequest.OpType.UPDATE : 只更新发送的字段
*
* @author xupeng
* @date: 2019-01-23
*/
public EsObject(String id, Object object, DocWriteRequest.OpType opType) {
this.id = id;
this.object = object;
this.opType = opType;
}
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public Object getObject() {
return object;
}
public void setObject(Object object) {
this.object = object;
}
public DocWriteRequest.OpType getOpType() {
return opType;
}
public void setOpType(DocWriteRequest.OpType opType) {
this.opType = opType;
}
}
suggest-task/src/main/java/com/secoo/so/suggest/es/EsSearchCallback.java 0 → 100644
package com.secoo.so.suggest.es;
import java.util.List;
import java.util.Map;
/**
*
* 分页查询时可以使用的回调接口
**/
public interface EsSearchCallback {
/**
* 分页查询时的callback，如果有该实现，search接口最终将返回空数据
* @param results
*/
public void callback(List<Map<String, Object>> results);
}
suggest-task/src/main/java/com/secoo/so/suggest/helper/AdvWordsHelper.java 0 → 100644
package com.secoo.so.suggest.helper;
import com.secoo.so.suggest.db.ErpDataSource;
import com.secoo.so.suggest.util.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.Statement;
import java.util.HashSet;
import java.util.Set;
/**
* 加载敏感词汇
*/
public class AdvWordsHelper {
private static final Logger LOG = LoggerFactory.getLogger(AdvWordsHelper.class);
/**
* 敏感词key
*/
public static final String ADV_WORDS = "so.advWords";
/**
* 汉字匹配
*/
public static final String regEx1 = "[\\u4e00-\\u9fa5]";
private static final String SENSITIVE_SQL = "SELECT sensitive_words FROM t_sensitive_info WHERE status=1 and del_flag=0";
/**
* 获取敏感词
*/
public static String getAdvWords() {
Connection conn = ErpDataSource.getConnection();
Statement stmt = null;
ResultSet rs = null;
Set<String> advWordSet = new HashSet<>();
try {
stmt = conn.createStatement();
rs = stmt.executeQuery(SENSITIVE_SQL);
while (rs.next()) {
String sensitiveWord = rs.getString(1);
if (StringUtils.isNotEmpty(sensitiveWord.trim())) {
advWordSet.add(sensitiveWord);
}
}
} catch (Exception e) {
LOG.error("获取敏感词异常: " + e.getMessage(), e);
} finally {
ObjectUtils.safeClose(conn, stmt, rs);
}
if (advWordSet.size() > 0) {
return StringUtils.join(advWordSet, ",");
}
return null;
}
}
suggest-task/src/main/java/com/secoo/so/suggest/task/SuggestTask.java 0 → 100644
package com.secoo.so.suggest.task;
import com.alibaba.fastjson.JSON;
import com.secoo.so.suggest.config.ConfigUtil;
import com.secoo.so.suggest.db.DwDataSource;
import com.secoo.so.suggest.db.ErpDataSource;
import com.secoo.so.suggest.entity.BrandInfo;
import com.secoo.so.suggest.entity.CategoryInfo;
import com.secoo.so.suggest.entity.EsSuggestKeywordInfo;
import com.secoo.so.suggest.entity.SearchKeywordInfo;
import com.secoo.so.suggest.es.EsClient;
import com.secoo.so.suggest.es.EsObject;
import com.secoo.so.suggest.util.*;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import java.io.File;
import java.io.Serializable;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
/**
* 初始化suggest搜索词到es索引
*/
@Slf4j
public class SuggestTask {
private static Map<String, Long> brandMap = new HashMap<>(); // 品牌
private static Map<String, Long> categoryMap = new HashMap<>(); // 分类
private static Map<String, Integer> manualMap = new HashMap<>(); // 人工干预
private static Map<String, Boolean> sensitiveMap = new HashMap<>(); // 敏感词
private static Map<String, Boolean> europeWordMap = new HashMap<>(); // 欧洲
private static List<String> prefixFilterList = new ArrayList<>(); // 前缀过滤列表
private static int maxTagSize = 5;
private static long startTime = System.currentTimeMillis();
public static void main(String[] args) {
startTime = System.currentTimeMillis();
log.info(">>>>>>>>>>>> start run SuggestTask , startTime: " + startTime);
// 初始化配置信息
ConfigUtil.init();
ConfigUtil.printAll();
prefixFilterList = loadPrefixFilterList();
maxTagSize = ConfigUtil.getInt("suggestTask.suggestTagMaxSize", 5);
// 加载品牌、品类信息
brandMap = loadBrandMap();
categoryMap = loadCategoryMap();
// 加载文件
manualMap = loadManualMap();
sensitiveMap = loadSensitiveMap();
europeWordMap = loadEuropeWordMap();
// 加载搜索词并处理
processSuggestTask(startTime);
log.info("<<<<<<<<<<<< end run SuggestTask, startTime: {} , cost: {}ms", startTime, (System.currentTimeMillis() - startTime) );
}
private static Map<String, Long> loadBrandMap() {
Map<String, Long> brandMap = new HashMap<>();
List<BrandInfo> brandList = ErpDataSource.queryBrandInfoList();
if (CollectionUtils.isNotEmpty(brandList)) {
for (BrandInfo brandInfo : brandList) {
putIfKeyNotBlank(brandMap, cleanKeyword(brandInfo.getEnName()), brandInfo.getId());
putIfKeyNotBlank(brandMap, cleanKeyword(brandInfo.getChName()), brandInfo.getId());
putIfKeyNotBlank(brandMap, cleanKeyword(brandInfo.getShortName()), brandInfo.getId());
if (StringUtils.isNotBlank(brandInfo.getNickName()) && brandInfo.getNickName().contains(",")) {
List<String> nickNameList = StringUtils.splitToList(brandInfo.getNickName(), ",");
if (CollectionUtils.isNotEmpty(nickNameList)) {
nickNameList.forEach(nickName -> {
if (StringUtils.isNotBlank(nickName)) {
putIfKeyNotBlank(brandMap, cleanKeyword(nickName), brandInfo.getId());
}
});
}
} else {
putIfKeyNotBlank(brandMap, cleanKeyword(brandInfo.getNickName()), brandInfo.getId());
}
}
}
return brandMap;
}
private static Map<String, Long> loadCategoryMap() {
Map<String, Long> categoryMap = new HashMap<>();
List<CategoryInfo> categoryInfoList = ErpDataSource.queryCategoryInfoList();
if (CollectionUtils.isNotEmpty(categoryInfoList)) {
for (CategoryInfo categoryInfo : categoryInfoList) {
putIfKeyNotBlank(categoryMap, cleanKeyword(categoryInfo.getName()), categoryInfo.getId());
}
}
return categoryMap;
}
private static Map<String, Integer> loadManualMap() {
Map<String, Integer> manualMap = new HashMap<>();
String folderPath = ConfigUtil.getString("suggestTask.ManualFolder");
if (StringUtils.isNotBlank(folderPath)) {
File folder = new File(folderPath);
if (folder.exists() && folder.isDirectory()) {
for (File file : folder.listFiles()) {
log.info("load loadManualMap file: " + file.getAbsolutePath());
List<String> lines = FileUtils.readLines(file);
if (CollectionUtils.isNotEmpty(lines)) {
for (String line : lines) {
if (StringUtils.isNotBlank(line)) {
String[] lineSplit = line.split("\\|");
if (StringUtils.isNotBlank(lineSplit[0])) {
Integer value = (lineSplit.length > 1 ? StringUtils.str2Int(lineSplit[1]) : 1);
manualMap.put(cleanKeyword(lineSplit[0]), value != null ? value : 1);
}
}
}
}
}
}
}
return manualMap;
}
private static Map<String, Boolean> loadSensitiveMap() {
Map<String, Boolean> sensitiveMap = new HashMap<>();
String folderPath = ConfigUtil.getString("suggestTask.SensitiveFolder");
if (StringUtils.isNotBlank(folderPath)) {
File folder = new File(folderPath);
if (folder.exists() && folder.isDirectory()) {
for (File file : folder.listFiles()) {
log.info("load loadSensitiveMap file: " + file.getAbsolutePath());
List<String> lines = FileUtils.readLines(file);
if (CollectionUtils.isNotEmpty(lines)) {
for (String line : lines) {
if (StringUtils.isNotBlank(line)) {
sensitiveMap.put(cleanKeyword(line), true);
}
}
}
}
}
}
return sensitiveMap;
}
private static Map<String, Boolean> loadEuropeWordMap() {
Map<String, Boolean> europeWordMap = new HashMap<>();
String folderPath = ConfigUtil.getString("suggestTask.EuropeWordFolder");
if (StringUtils.isNotBlank(folderPath)) {
File folder = new File(folderPath);
if (folder.exists() && folder.isDirectory()) {
for (File file : folder.listFiles()) {
log.info("load loadEuropeWordMap file: " + file.getAbsolutePath());
List<String> lines = FileUtils.readLines(file);
if (CollectionUtils.isNotEmpty(lines)) {
for (String line : lines) {
if (StringUtils.isNotBlank(line)) {
europeWordMap.put(cleanKeyword(line), true);
}
}
}
}
}
}
return europeWordMap;
}
public static List<String> loadPrefixFilterList() {
List<String> prefixFilterList = new ArrayList<>();
String val = ConfigUtil.getString("suggestTask.prefixFilterList");
if (StringUtils.isNotBlank(val)) {
prefixFilterList = JSON.parseArray(val, String.class);
}
return prefixFilterList;
}
private static String cleanKeyword(String keyword) {
if (keyword != null) {
String fixKeyword = PinYinUtils.convertToSimplifiedChinese(keyword);
fixKeyword = StringUtils.strip(StringUtils.dbc2Sbc(fixKeyword), " \ufffc,，.。");
fixKeyword = StringUtils.cleanMultiBlank(fixKeyword);
return fixKeyword.toLowerCase();
}
return null;
}
private static void putIfKeyNotBlank(Map<String, Long> map, String key, Long value) {
if (map != null && StringUtils.isNotBlank(key)) {
map.put(key, value);
}
}
/**
* 处理suggest-task任务
*/
private static void processSuggestTask(long startTime) {
// 查询搜索词数量和最大id
Map<String, Long> countResultMap = DwDataSource.querySearchWordCountAndMaxId();
Long count = countResultMap.get("count");
Long maxId = countResultMap.get("maxId");
Long minId = countResultMap.get("minId");
log.info("querySearchWordCountAndMaxId: count={}, maxId={}", count, maxId);
Long warningCount = ConfigUtil.getLong("suggestTask.searchWordWarningCount", 1000000);
if (count < warningCount) {
log.warn("search word count is too little: count={}, warningCount={}, send warning", count, warningCount);
FeiShuUtil.sendMessage("suggest-task异常", "搜索词数量过低，不执行索引");
return;
}
ConcurrentHashMap<String, EsSuggestKeywordInfo> esSuggestKeywordMap = new ConcurrentHashMap<>();
// 通过线程池分批次并发处理搜索词
long batchSize = ConfigUtil.getLong("suggestTask.batchSize", 10000);
int threadPoolSize = ConfigUtil.getInt("suggestTask.threadPoolSize", 10);
ExecutorService execThreadPool = Executors.newFixedThreadPool(threadPoolSize);
int taskCount = 0;
for (long startId = minId; startId <= maxId; startId = startId + batchSize) {
taskCount++;
execThreadPool.submit(new SearchKeywordProcessTask(esSuggestKeywordMap, startId, startId + batchSize, startTime));
}
execThreadPool.shutdown();
while (true) {
if (execThreadPool.isTerminated()) {
log.info("所有的子线程任务 {}个 都结束了, 关闭线程池成功", taskCount);
break;
}
ObjectUtils.safeSleep(5000);
}
// 将品牌词、品类词、人工干预词都写入es
Set<String> fillKeywordSet = new HashSet<>();
fillKeywordSet.addAll(brandMap.keySet());
fillKeywordSet.addAll(categoryMap.keySet());
fillKeywordSet.addAll(manualMap.keySet());
for (String fillKeyword : fillKeywordSet) {
if (!esSuggestKeywordMap.containsKey(fillKeyword)) {
esSuggestKeywordMap.put(fillKeyword, buildDefaultEsSuggestKeywordInfo(fillKeyword));
}
}
// 过滤词
List<EsSuggestKeywordInfo> suggestKeywordInfoList = new ArrayList<>();
int processCount = 0;
int totalCount = esSuggestKeywordMap.values().size();
for (EsSuggestKeywordInfo suggestKeywordInfo : esSuggestKeywordMap.values()) {
// 不过滤的suggest词写es
if (!isFilterSuggestKeyword(suggestKeywordInfo)) {
// 转拼音
suggestKeywordInfo.setKeywordPinYin(PinYinUtils.changeToWithoutTonePinYin(suggestKeywordInfo.getKeyword(), ""));
// 保存es前执行标签清洗
cleanBeforeSaveToEs(suggestKeywordInfo);
suggestKeywordInfoList.add(suggestKeywordInfo);
}
processCount++;
if (processCount > 0 && processCount % 1000 == 0 || processCount == totalCount) {
log.info("keyword filter process: {} / {}", processCount, totalCount);
}
}
if ("true".equalsIgnoreCase(System.getProperty("suggest.saveToFile"))) {
// save to file
saveSuggestKeywordToFile(suggestKeywordInfoList);
} else {
// save to es
saveSuggestKeywordToEs(suggestKeywordInfoList);
}
}
private static EsSuggestKeywordInfo buildDefaultEsSuggestKeywordInfo(String keyword) {
EsSuggestKeywordInfo esSuggestKeywordInfo = new EsSuggestKeywordInfo();
esSuggestKeywordInfo.setKeyword(keyword);
esSuggestKeywordInfo.setKeywordVersion(DateUtils.formatDate(startTime, "yyyy-MM-dd"));
esSuggestKeywordInfo.setUpdateTime(startTime);
esSuggestKeywordInfo.setYearCount(0);
esSuggestKeywordInfo.setYearClickCount(0);
esSuggestKeywordInfo.setYearCartCount(0);
esSuggestKeywordInfo.setWeekCount(0L);
esSuggestKeywordInfo.setWeekClickCount(0L);
esSuggestKeywordInfo.setWeekCartCount(0L);
SearchKeywordInfo searchKeywordInfo = new SearchKeywordInfo();
searchKeywordInfo.setId(0L);
searchKeywordInfo.setKeyword(keyword);
searchKeywordInfo.setPrepareTags(null);
searchKeywordInfo.setYearPv(0);
searchKeywordInfo.setYearProductClickCount(0);
searchKeywordInfo.setYearAddCartCount(0);
searchKeywordInfo.setWeekPv(0L);
searchKeywordInfo.setWeekProductClickCount(0L);
searchKeywordInfo.setWeekAddCartCount(0L);
searchKeywordInfo.setWeekUv(0L);
searchKeywordInfo.setWeekProductClickUv(0L);
searchKeywordInfo.setWeekAddCartUv(0L);
searchKeywordInfo.setMonthPv(0L);
searchKeywordInfo.setMonthProductClickCount(0L);
searchKeywordInfo.setMonthAddCartCount(0L);
searchKeywordInfo.setMonthUv(0L);
searchKeywordInfo.setMonthProductClickUv(0L);
searchKeywordInfo.setMonthAddCartUv(0L);
searchKeywordInfo.setPDay(DateUtils.formatDate(startTime, "yyyy-MM-dd"));
// 计算suggestKeyword权重等属性
processEsSuggestKeywordInfo(esSuggestKeywordInfo, searchKeywordInfo);
return esSuggestKeywordInfo;
}
/**
* 处理搜索词
*/
private static void processSearchKeyword(ConcurrentHashMap<String, EsSuggestKeywordInfo> esSuggestKeywordMap, List<SearchKeywordInfo> searchKeywordInfoList, long startTime) {
if (CollectionUtils.isNotEmpty(searchKeywordInfoList)) {
for (SearchKeywordInfo searchKeywordInfo : searchKeywordInfoList) {
if (StringUtils.isNotBlank(searchKeywordInfo.getKeyword())) {
String keyword = cleanKeyword(searchKeywordInfo.getKeyword());
synchronized (keyword) {
EsSuggestKeywordInfo suggestKeywordInfo = esSuggestKeywordMap.get(keyword);
if (suggestKeywordInfo == null) {
suggestKeywordInfo = new EsSuggestKeywordInfo();
suggestKeywordInfo.setKeyword(keyword);
suggestKeywordInfo.setYearCount(searchKeywordInfo.getYearPv());
suggestKeywordInfo.setYearClickCount(searchKeywordInfo.getYearProductClickCount());
suggestKeywordInfo.setYearCartCount(searchKeywordInfo.getYearAddCartCount());
suggestKeywordInfo.setWeekCount(searchKeywordInfo.getWeekPv());
suggestKeywordInfo.setWeekClickCount(searchKeywordInfo.getWeekProductClickCount());
suggestKeywordInfo.setWeekCartCount(searchKeywordInfo.getWeekAddCartCount());
suggestKeywordInfo.setSuggestTags(searchKeywordInfo.getPrepareTags());
suggestKeywordInfo.setKeywordVersion(searchKeywordInfo.getPDay());
suggestKeywordInfo.setUpdateTime(startTime);
esSuggestKeywordMap.put(keyword, suggestKeywordInfo);
} else {
suggestKeywordInfo.setYearCount(suggestKeywordInfo.getYearCount() + searchKeywordInfo.getYearPv());
suggestKeywordInfo.setYearClickCount(suggestKeywordInfo.getYearClickCount() + searchKeywordInfo.getYearProductClickCount());
suggestKeywordInfo.setYearCartCount(suggestKeywordInfo.getYearCartCount() + searchKeywordInfo.getYearAddCartCount());
suggestKeywordInfo.setWeekCount(suggestKeywordInfo.getWeekCount() + searchKeywordInfo.getWeekPv());
suggestKeywordInfo.setWeekClickCount(suggestKeywordInfo.getWeekClickCount() + searchKeywordInfo.getWeekProductClickCount());
suggestKeywordInfo.setWeekCartCount(suggestKeywordInfo.getWeekCartCount() + searchKeywordInfo.getWeekAddCartCount());
if (StringUtils.isBlank(suggestKeywordInfo.getSuggestTags()) && "null".equalsIgnoreCase(suggestKeywordInfo.getSuggestTags())) {
suggestKeywordInfo.setSuggestTags(suggestKeywordInfo.getSuggestTags());
}
}
// 计算suggestKeyword权重等属性
processEsSuggestKeywordInfo(suggestKeywordInfo, searchKeywordInfo);
}
}
}
}
}
/**
* 保存到es
*/
private static void saveSuggestKeywordToEs(List<EsSuggestKeywordInfo> suggestKeywordInfoList) {
log.info("start saveSuggestKeywordToEs");
if (CollectionUtils.isNotEmpty(suggestKeywordInfoList)) {
String esUrl = ConfigUtil.getString("suggestTask.es.url");
String esUser = ConfigUtil.getString("suggestTask.es.user");
String esPassword = ConfigUtil.getString("suggestTask.es.password");
String esIndex = ConfigUtil.getString("suggestTask.es.index");
String esType = ConfigUtil.getString("suggestTask.es.type");
int esBatchSize = ConfigUtil.getInt("suggestTask.es.batchSize", 2000);
EsClient esClient = EsClient.buildEsClient(esUrl, esUser, esPassword);
try {
List<List<EsSuggestKeywordInfo>> subLists = CollectionUtils.splitList(suggestKeywordInfoList, esBatchSize);
for (List<EsSuggestKeywordInfo> subList : subLists) {
List<EsObject> esList = new ArrayList<>();
for (EsSuggestKeywordInfo esSuggestKeywordInfo : subList) {
esList.add(new EsObject(StringUtils.md5(esSuggestKeywordInfo.getKeyword()), esSuggestKeywordInfo));
}
try {
esClient.batch(esIndex, esType, esList);
} catch (Exception e) {
log.error("saveSuggestKeywordToEs error", e);
FeiShuUtil.sendMessage("suggest-task save to es 异常", "suggest-task save to es 异常");
}
}
} finally {
esClient.close();
}
}
}
private static void saveSuggestKeywordToFile(List<EsSuggestKeywordInfo> suggestKeywordInfoList) {
log.info("start saveSuggestKeywordToFile");
if (CollectionUtils.isNotEmpty(suggestKeywordInfoList)) {
int batch = 2000;
String fileName = "/tmp/suggest-task/suggest-index-" + DateUtils.formatDate(startTime, "yyyyMMddHHmmss") + ".json";
List<String> lines = new ArrayList<>();
int count = 0;
StringBuilder keywordBuilder = new StringBuilder();
for (count = 0; count < suggestKeywordInfoList.size(); count++) {
EsSuggestKeywordInfo suggestKeywordInfo = suggestKeywordInfoList.get(count);
lines.add(JSON.toJSONString(suggestKeywordInfo));
keywordBuilder.append(suggestKeywordInfo.getKeyword()).append("\n");
if (count > 0 && lines.size() % batch == 0) {
log.info("save {}/{} result to file: {}", lines.size(), count, fileName);
FileUtils.saveToFile(lines, fileName, true);
lines = new ArrayList<>();
}
}
if (CollectionUtils.isNotEmpty(lines)) {
log.info("save {}/{} result to file: {}", lines.size(), count, fileName);
FileUtils.saveToFile(lines, fileName, true);
lines.clear();
}
String keywordFileName = "/tmp/suggest-task/suggest-index-keyword-" + DateUtils.formatDate(startTime, "yyyyMMddHHmmss") + ".txt";
FileUtils.saveToFile(keywordBuilder.toString(), keywordFileName, false);
}
}
/**
* 是否需要过滤掉suggest关键词
*/
private static boolean isFilterSuggestKeyword(EsSuggestKeywordInfo suggestKeywordInfo) {
// 过滤掉太短、太长的词
if (StringUtils.isBlank(suggestKeywordInfo.getKeyword())
|| suggestKeywordInfo.getKeyword().length() <= 1
|| StringUtils.getByteLength(suggestKeywordInfo.getKeyword()) > 50) {
return true;
}
// 前缀过滤
for (String prefix : prefixFilterList) {
if (suggestKeywordInfo.getKeyword().startsWith(prefix)) {
return true;
}
}
// 敏感词过滤
if (suggestKeywordInfo.getIsSensitive()) {
return true;
}
// 品牌词 类目词 人工干预词 不做过滤
if (suggestKeywordInfo.getIsBrand() || suggestKeywordInfo.getIsCategory() || suggestKeywordInfo.getIsManual()) {
suggestKeywordInfo.setIsSensitive(false);
return false;
}
// 过滤掉纯数字的搜索词，原：过滤掉商品id,商品id是有7位数字组成
if (suggestKeywordInfo.getKeyword().length() > 6 && StringUtils.isNumber(suggestKeywordInfo.getKeyword())) {
return true;
}
// 年数据过滤
if (suggestKeywordInfo.getYearCount() < 2 || suggestKeywordInfo.getYearClickCount() < 2) {
return true;
}
if (isHotSearchWord(suggestKeywordInfo) // 判断是否是热搜词 一年内搜索次数大于50或者一周内搜索次数大于5
|| (suggestKeywordInfo.getYearCount() > 5 && isHighCartRatio(suggestKeywordInfo)) // 搜索次数比较多 转化率或者点击率较高的 不过滤
|| isHighClickRatio(suggestKeywordInfo)) { // 搜索次数不多 但是转化率很高的 或者有加购 不过滤
return false;
}
return true;
}
private static void cleanBeforeSaveToEs(EsSuggestKeywordInfo suggestKeywordInfo) {
if (suggestKeywordInfo != null) {
if (suggestKeywordInfo.getSuggestTags() == null || "null".equalsIgnoreCase(suggestKeywordInfo.getSuggestTags())) {
suggestKeywordInfo.setSuggestTags("");
return;
}
List<String> tagList = StringUtils.splitToList(suggestKeywordInfo.getSuggestTags(), ",");
if (tagList != null && tagList.size() > maxTagSize) {
suggestKeywordInfo.setSuggestTags(StringUtils.join(CollectionUtils.subList(tagList, 0, maxTagSize), ","));
}
}
}
private static boolean isHotSearchWord(EsSuggestKeywordInfo suggestKeywordInfo) {
return suggestKeywordInfo.getYearCount() > 50 || suggestKeywordInfo.getWeekCount() > 5;
}
private static boolean isHighCartRatio(EsSuggestKeywordInfo suggestKeywordInfo) {
return suggestKeywordInfo.getYearCartRatio() > 0.025 || suggestKeywordInfo.getWeekCartRatio() > 0.025
|| suggestKeywordInfo.getYearClickRatio() > 0.1 || suggestKeywordInfo.getWeekClickRatio() > 0.1;
}
private static boolean isHighClickRatio(EsSuggestKeywordInfo suggestKeywordInfo) {
if (suggestKeywordInfo.getYearCount() < 5 && suggestKeywordInfo.getYearClickRatio() < 0.6 && suggestKeywordInfo.getYearCartCount() == 0) {
return false;
}
return suggestKeywordInfo.getYearClickRatio() > 0.2 || suggestKeywordInfo.getWeekClickRatio() > 0.2 || suggestKeywordInfo.getYearCartCount() >= 1;
}
private static void processEsSuggestKeywordInfo(EsSuggestKeywordInfo suggestKeywordInfo, SearchKeywordInfo searchKeywordInfo) {
String keyword = suggestKeywordInfo.getKeyword();
suggestKeywordInfo.setIsBrand(brandMap.containsKey(keyword));
suggestKeywordInfo.setIsCategory(categoryMap.containsKey(keyword));
suggestKeywordInfo.setIsSensitive(sensitiveMap.containsKey(keyword));
suggestKeywordInfo.setIsEuropeWord(europeWordMap.containsKey(keyword));
suggestKeywordInfo.setIsManual(manualMap.containsKey(keyword));
suggestKeywordInfo.setManualValue(suggestKeywordInfo.getIsManual() ? manualMap.get(keyword) : 0);
// 年点击加购率
suggestKeywordInfo.setYearClickRatio(CalculateUtils.calculateRatio(suggestKeywordInfo.getYearClickCount(), suggestKeywordInfo.getYearCount()));
suggestKeywordInfo.setYearCartRatio(CalculateUtils.calculateRatio(suggestKeywordInfo.getYearCartCount(), suggestKeywordInfo.getYearCount()));
// 周点击加购率
suggestKeywordInfo.setWeekClickRatio(CalculateUtils.calculateRatio(suggestKeywordInfo.getWeekClickCount().intValue(), suggestKeywordInfo.getWeekCount().intValue()));
suggestKeywordInfo.setWeekCartRatio(CalculateUtils.calculateRatio(suggestKeywordInfo.getWeekCartCount().intValue(), suggestKeywordInfo.getWeekCount().intValue()));
// 年加购率 再加权
if (suggestKeywordInfo.getYearCount() != 0 && suggestKeywordInfo.getYearCartCount() != 0) {
suggestKeywordInfo.setYearCartRatio(suggestKeywordInfo.getYearCartRatio() * 3);
}
// 周加购率 再加权
if (suggestKeywordInfo.getWeekCount() != 0 && suggestKeywordInfo.getWeekCartCount() != 0) {
suggestKeywordInfo.setWeekCartRatio(suggestKeywordInfo.getWeekCartRatio() * 3);
}
// 周点击率 再加权
if (suggestKeywordInfo.getWeekCount() != 0 && suggestKeywordInfo.getWeekClickCount() != 0) {
suggestKeywordInfo.setWeekClickRatio(suggestKeywordInfo.getWeekClickRatio() * 2);
}
calculateWordRank(suggestKeywordInfo);
calculateWordABRank(suggestKeywordInfo, searchKeywordInfo);
addNewScoreIfNewHotWord(suggestKeywordInfo);
}
public static Double calculateWordRank(EsSuggestKeywordInfo suggestKeywordInfo) {
Double wordRank = 10000.0;
// 长度因子
wordRank += 3000 * CalculateUtils.calculateLengthFactor(StringUtils.getByteLength(suggestKeywordInfo.getKeyword()));
// 年数量因子
wordRank += 2000 * CalculateUtils.calculateCountFactor(suggestKeywordInfo.getYearCount(), 1);
// 周数量因子
wordRank += 2000 * CalculateUtils.calculateCountFactor(suggestKeywordInfo.getWeekCount().intValue(), 52);
// 年点击率因子
wordRank += 3000 * CalculateUtils.calculateRatioFactor(suggestKeywordInfo.getYearClickRatio(), suggestKeywordInfo.getYearClickCount());
// 周点击率因子
wordRank += 3000 * CalculateUtils.calculateRatioFactor(suggestKeywordInfo.getWeekClickRatio().doubleValue(), suggestKeywordInfo.getWeekClickCount().intValue());
// 年加购率因子
wordRank += 3000 * CalculateUtils.calculateRatioFactor(suggestKeywordInfo.getYearCartRatio().doubleValue(), suggestKeywordInfo.getYearCartCount());
// 周加购率因子
wordRank += 3000 * CalculateUtils.calculateRatioFactor(suggestKeywordInfo.getWeekCartRatio(), suggestKeywordInfo.getWeekCartCount().intValue());
if (suggestKeywordInfo.getIsBrand()) {
wordRank *= 1.8;
}
if (suggestKeywordInfo.getIsCategory()) {
wordRank *= 1.2;
}
if (suggestKeywordInfo.getIsManual() && suggestKeywordInfo.getManualValue() > 0) {
wordRank *= Math.sqrt(suggestKeywordInfo.getManualValue() * 1.0);
}
suggestKeywordInfo.setWordRank(wordRank);
return wordRank;
}
public static Double calculateWordABRank(EsSuggestKeywordInfo suggestKeywordInfo, SearchKeywordInfo searchKeywordInfo) {
// 月点击加购率
Double monthClickRatio = CalculateUtils.calculateRatio(searchKeywordInfo.getMonthProductClickUv().intValue(), searchKeywordInfo.getMonthUv().intValue());
Double monthCartRatio = CalculateUtils.calculateRatio(searchKeywordInfo.getMonthAddCartUv().intValue(), searchKeywordInfo.getMonthUv().intValue());
// 周点击加购率(和A相比, count 换成了uv)
Double weekClickRatioNew = CalculateUtils.calculateRatio(searchKeywordInfo.getWeekProductClickUv().intValue(), searchKeywordInfo.getWeekUv().intValue());
Double weekCartRatioNew = CalculateUtils.calculateRatio(searchKeywordInfo.getWeekAddCartUv().intValue(), searchKeywordInfo.getWeekUv().intValue());
// 月点击
if (searchKeywordInfo.getMonthProductClickUv() != 0 && searchKeywordInfo.getMonthUv() != 0) {
monthClickRatio *= 1.5;
}
// 月加购，加权
if (searchKeywordInfo.getMonthAddCartUv() != 0 && searchKeywordInfo.getMonthUv() != 0) {
monthCartRatio *= 3;
}
// 周点击，加权
if (searchKeywordInfo.getWeekProductClickUv() != 0 && searchKeywordInfo.getWeekUv() != 0) {
weekClickRatioNew *= 2;
}
// 周加购，加权
if (searchKeywordInfo.getWeekAddCartUv() != 0 && searchKeywordInfo.getWeekUv() != 0) {
weekCartRatioNew *= 3;
}
Double wordABRank = 10000.0;
// 长度因子
wordABRank += 3000 * CalculateUtils.calculateLengthFactor(StringUtils.getByteLength(suggestKeywordInfo.getKeyword()));
// 月数量因子
wordABRank += 2000 * CalculateUtils.calculateCountFactor(searchKeywordInfo.getMonthUv().intValue(), 4);
// 周数量因子
wordABRank += 2000 * CalculateUtils.calculateCountFactor(searchKeywordInfo.getWeekUv().intValue(), 52);
// 年数量因子
wordABRank += 2000 * CalculateUtils.calculateCountFactor(suggestKeywordInfo.getYearCount(), 1);
// 点击
// 月点击率因子
wordABRank += 3000 * CalculateUtils.calculateRatioFactor(monthClickRatio, searchKeywordInfo.getMonthProductClickUv().intValue());
// 周点击率因子
wordABRank += 3000 * CalculateUtils.calculateRatioFactor(weekClickRatioNew, searchKeywordInfo.getWeekUv().intValue());
// 加购
// 年加购率因子
wordABRank += 3000 * CalculateUtils.calculateRatioFactor(suggestKeywordInfo.getYearCartRatio().doubleValue(), suggestKeywordInfo.getYearCartCount());
// 月加购率因子
wordABRank += 3000 * CalculateUtils.calculateRatioFactor(monthCartRatio, searchKeywordInfo.getMonthUv().intValue());
// 周加购率因子
wordABRank += 3000 * CalculateUtils.calculateRatioFactor(weekCartRatioNew, searchKeywordInfo.getWeekUv().intValue());
if (suggestKeywordInfo.getIsBrand()) {
wordABRank *= 1.8;
}
if (suggestKeywordInfo.getIsCategory()) {
wordABRank *= 1.2;
}
if (suggestKeywordInfo.getIsManual() && suggestKeywordInfo.getManualValue() > 0) {
wordABRank *= Math.sqrt(suggestKeywordInfo.getManualValue() * 1.0);
}
suggestKeywordInfo.setWordABRank(wordABRank);
return wordABRank;
}
private static void addNewScoreIfNewHotWord(EsSuggestKeywordInfo suggestKeywordInfo) {
// 比例有意义
if (suggestKeywordInfo.getWeekCount() == 0 || suggestKeywordInfo.getYearCount() == 0 || suggestKeywordInfo.getWeekCount() < 20) {
return;
}
// 周点击占年点击 40% 以上
if (suggestKeywordInfo.getWeekCount() * 10 / suggestKeywordInfo.getYearCount() <= 5) {
return;
}
if (suggestKeywordInfo.getWeekClickCount() < 3 || suggestKeywordInfo.getWeekCount() < 5) {
return;
}
// 新词加分大小 类似于 人工干预值
suggestKeywordInfo.setWordABRank(new Double(suggestKeywordInfo.getWordABRank() * Math.sqrt(5.0)));
}
@Data
static class SearchKeywordProcessTask implements Runnable, Serializable {
private static final long serialVersionUID = -2853856815712590673L;
public SearchKeywordProcessTask(ConcurrentHashMap<String, EsSuggestKeywordInfo> esSuggestKeywordMap, Long startId, Long endId, Long startTime) {
this.esSuggestKeywordMap = esSuggestKeywordMap;
this.startId = startId;
this.endId = endId;
this.startTime = startTime;
}
private ConcurrentHashMap<String, EsSuggestKeywordInfo> esSuggestKeywordMap;
private Long startId;
private Long endId;
private Long startTime;
@Override
public void run() {
List<SearchKeywordInfo> searchKeywordInfoList = DwDataSource.querySearchKeywordInfoList(startId, endId);
log.info("start process startId:{}, endId:{}, count:{}", startId, endId, searchKeywordInfoList.size());
if (CollectionUtils.isNotEmpty(searchKeywordInfoList)) {
processSearchKeyword(this.esSuggestKeywordMap, searchKeywordInfoList, startTime);
}
}
}
}
suggest-task/src/main/java/com/secoo/so/suggest/util/CalculateUtils.java 0 → 100644
package com.secoo.so.suggest.util;
/**
* @author xupeng
* @date: 2022/1/27
*/
public class CalculateUtils {
public static Double calculateRatio(Integer numerator, Integer denominator) {
if (numerator == null || numerator == 0 || denominator == null || numerator == 0) {
return 0D;
}
return numerator.doubleValue() / denominator.doubleValue();
}
public static Double calculateLengthFactor(Integer length) {
//根据文本长度转换为长度因子
return 1.0 / new Double(2 * length + 1);
}
public static Double calculateRatioFactor(Double ratio, Integer count) {
Double rank = 1.0;
if (count > 1 && count < 10) {
rank = 1.2;
} else if (count >= 10 && count < 20) {
rank = 1.4;
} else if (count >= 20 && count < 50) {
rank = 1.6;
} else if (count >= 50 && count < 100) {
rank = 1.8;
} else if (count >= 100 && count < 200) {
rank = 2.0;
} else if (count >= 200 && count < 500) {
rank = 2.2;
} else if (count >= 500) {
rank = 2.5;
}
//根据搜索转化率，转换为热度因子
return Math.log10(Math.sqrt(ratio + 10)) * rank;
}
public static Double calculateCountFactor(Integer count, Integer rank) {
//根据搜索次数，转换为热度因子
count = count * rank + 10;
return Math.log10(Math.sqrt(new Double(count)));
}
}
suggest-task/src/main/java/com/secoo/so/suggest/util/CollectionUtils.java 0 → 100644
package com.secoo.so.suggest.util;
import java.util.*;
/**
* Created with IntelliJ IDEA. User: Administrator Date: 13-11-11 Time: 下午12:24
* To change this template use File | Settings | File Templates.
*/
public abstract class CollectionUtils {
/**
* Return <code>true</code> if the supplied Collection is <code>null</code>
* or empty. Otherwise, return <code>false</code>.
*
* @param collection the Collection to check
* @return whether the given Collection is empty
*/
@SuppressWarnings("rawtypes")
public static boolean isEmpty(Collection collection) {
return (collection == null || collection.isEmpty());
}
@SuppressWarnings("rawtypes")
public static boolean isNotEmpty(Collection collection) {
return (collection != null && !collection.isEmpty());
}
/**
* Return <code>true</code> if the supplied Map is <code>null</code> or
* empty. Otherwise, return <code>false</code>.
*
* @param map the Map to check
* @return whether the given Map is empty
*/
@SuppressWarnings("rawtypes")
public static boolean isEmpty(Map map) {
return (map == null || map.isEmpty());
}
@SuppressWarnings("rawtypes")
public static boolean isNotEmpty(Map map) {
return (map != null && !map.isEmpty());
}
/**
* Convert the supplied array into a List. A primitive array gets converted
* into a List of the appropriate wrapper type.
* <p>
* A <code>null</code> source value will be converted to an empty List.
*
* @param source the (potentially primitive) array
* @return the converted List result
*/
@SuppressWarnings("rawtypes")
public static List arrayToList(Object source) {
return Arrays.asList(ObjectUtils.toObjectArray(source));
}
/**
* Merge the given array into the given Collection.
*
* @param array the array to merge (may be <code>null</code>)
* @param collection the target Collection to merge the array into
*/
@SuppressWarnings({"unchecked", "rawtypes"})
public static void mergeArrayIntoCollection(Object array, Collection collection) {
if (collection == null) {
throw new IllegalArgumentException("Collection must not be null");
}
Object[] arr = ObjectUtils.toObjectArray(array);
for (Object elem : arr) {
collection.add(elem);
}
}
/**
* Merge the given Properties instance into the given Map, copying all
* properties (key-value pairs) over.
* <p>
* Uses <code>Properties.propertyNames()</code> to even catch default
* properties linked into the original Properties instance.
*
* @param props the Properties instance to merge (may be <code>null</code>)
* @param map the target Map to merge the properties into
*/
@SuppressWarnings({"unchecked", "rawtypes"})
public static void mergePropertiesIntoMap(Properties props, Map map) {
if (map == null) {
throw new IllegalArgumentException("Map must not be null");
}
if (props != null) {
for (Enumeration en = props.propertyNames(); en.hasMoreElements(); ) {
String key = (String) en.nextElement();
Object value = props.getProperty(key);
if (value == null) {
// Potentially a non-String value...
value = props.get(key);
}
map.put(key, value);
}
}
}
/**
* Check whether the given Iterator contains the given element.
*
* @param iterator the Iterator to check
* @param element the element to look for
* @return <code>true</code> if found, <code>false</code> else
*/
@SuppressWarnings("rawtypes")
public static boolean contains(Iterator iterator, Object element) {
if (iterator != null) {
while (iterator.hasNext()) {
Object candidate = iterator.next();
if (ObjectUtils.nullSafeEquals(candidate, element)) {
return true;
}
}
}
return false;
}
/**
* Check whether the given Enumeration contains the given element.
*
* @param enumeration the Enumeration to check
* @param element the element to look for
* @return <code>true</code> if found, <code>false</code> else
*/
@SuppressWarnings("rawtypes")
public static boolean contains(Enumeration enumeration, Object element) {
if (enumeration != null) {
while (enumeration.hasMoreElements()) {
Object candidate = enumeration.nextElement();
if (ObjectUtils.nullSafeEquals(candidate, element)) {
return true;
}
}
}
return false;
}
/**
* Check whether the given Collection contains the given element instance.
* <p>
* Enforces the given instance to be present, rather than returning
* <code>true</code> for an equal element as well.
*
* @param collection the Collection to check
* @param element the element to look for
* @return <code>true</code> if found, <code>false</code> else
*/
@SuppressWarnings("rawtypes")
public static boolean containsInstance(Collection collection, Object element) {
if (collection != null) {
for (Object candidate : collection) {
if (candidate == element) {
return true;
}
}
}
return false;
}
/**
* Return <code>true</code> if any element in '<code>candidates</code>' is
* contained in ' <code>source</code>'; otherwise returns
* <code>false</code>.
*
* @param source the source Collection
* @param candidates the candidates to search for
* @return whether any of the candidates has been found
*/
@SuppressWarnings("rawtypes")
public static boolean containsAny(Collection source, Collection candidates) {
if (isEmpty(source) || isEmpty(candidates)) {
return false;
}
for (Object candidate : candidates) {
if (source.contains(candidate)) {
return true;
}
}
return false;
}
/**
* Return the first element in '<code>candidates</code>' that is contained
* in '<code>source</code> '. If no element in '<code>candidates</code>' is
* present in '<code>source</code>' returns <code>null</code>. Iteration
* order is {@link Collection} implementation specific.
*
* @param source the source Collection
* @param candidates the candidates to search for
* @return the first present object, or <code>null</code> if not found
*/
@SuppressWarnings("rawtypes")
public static Object findFirstMatch(Collection source, Collection candidates) {
if (isEmpty(source) || isEmpty(candidates)) {
return null;
}
for (Object candidate : candidates) {
if (source.contains(candidate)) {
return candidate;
}
}
return null;
}
/**
* Find a single value of the given type in the given Collection.
*
* @param collection the Collection to search
* @param type the type to look for
* @return a value of the given type found if there is a clear match, or
* <code>null</code> if none or more than one such value found
*/
@SuppressWarnings("unchecked")
public static <T> T findValueOfType(Collection<?> collection, Class<T> type) {
if (isEmpty(collection)) {
return null;
}
T value = null;
for (Object element : collection) {
if (type == null || type.isInstance(element)) {
if (value != null) {
// More than one value found... no clear single value.
return null;
}
value = (T) element;
}
}
return value;
}
/**
* Find a single value of one of the given types in the given Collection:
* searching the Collection for a value of the first type, then searching
* for a value of the second type, etc.
*
* @param collection the collection to search
* @param types the types to look for, in prioritized order
* @return a value of one of the given types found if there is a clear
* match, or <code>null</code> if none or more than one such value
* found
*/
public static Object findValueOfType(Collection<?> collection, Class<?>[] types) {
if (isEmpty(collection) || ObjectUtils.isEmpty(types)) {
return null;
}
for (Class<?> type : types) {
Object value = findValueOfType(collection, type);
if (value != null) {
return value;
}
}
return null;
}
/**
* Determine whether the given Collection only contains a single unique
* object.
*
* @param collection the Collection to check
* @return <code>true</code> if the collection contains a single reference
* or multiple references to the same instance, <code>false</code>
* else
*/
@SuppressWarnings("rawtypes")
public static boolean hasUniqueObject(Collection collection) {
if (isEmpty(collection)) {
return false;
}
boolean hasCandidate = false;
Object candidate = null;
for (Object elem : collection) {
if (!hasCandidate) {
hasCandidate = true;
candidate = elem;
} else if (candidate != elem) {
return false;
}
}
return true;
}
/**
* Find the common element type of the given Collection, if any.
*
* @param collection the Collection to check
* @return the common element type, or <code>null</code> if no clear common
* type has been found (or the collection was empty)
*/
@SuppressWarnings("rawtypes")
public static Class<?> findCommonElementType(Collection collection) {
if (isEmpty(collection)) {
return null;
}
Class<?> candidate = null;
for (Object val : collection) {
if (val != null) {
if (candidate == null) {
candidate = val.getClass();
} else if (candidate != val.getClass()) {
return null;
}
}
}
return candidate;
}
/**
* Marshal the elements from the given enumeration into an array of the
* given type. Enumeration elements must be assignable to the type of the
* given array. The array returned will be a different instance than the
* array given.
*/
public static <A, E extends A> A[] toArray(Enumeration<E> enumeration, A[] array) {
ArrayList<A> elements = new ArrayList<A>();
while (enumeration.hasMoreElements()) {
elements.add(enumeration.nextElement());
}
return elements.toArray(array);
}
/**
* 将值放到V=List的Map中
*
* @param result
* @param key
* @param t
* @param <T> List的对象
* @param <A> key的对象
*/
public static <T, A> void putValueToMapWithList(Map<A, List<T>> result, A key, T t, boolean checkValueExistsInList) {
if (result.containsKey(key)) {
List<T> list = result.get(key);
if (checkValueExistsInList) {
if (list.contains(t)) {
return;
}
}
list.add(t);
} else {
List<T> tmps = new ArrayList<>();
tmps.add(t);
result.put(key, tmps);
}
}
public static <T, A> void putValueToMapWithList(Map<A, List<T>> result, A key, T t) {
putValueToMapWithList(result, key, t, false);
}
/**
* 当Key在Map中不存在的时候，将值添加到Map中
*
* @param result
* @param key
* @param t
* @param <T>
* @param <A>
*/
public static <T, A> void putValueToMapIfNotExists(Map<A, T> result, A key, T t) {
if (result.containsKey(key)) {
return;
}
result.put(key, t);
}
/**
* 将列表值放到V=List的Map中
*
* @param result
* @param key
* @param ts
* @param <T>
*/
public static <T> void putValuesToMapWithList(Map<String, List<T>> result, String key, List<T> ts) {
if (result.containsKey(key)) {
result.get(key).addAll(ts);
} else {
result.put(key, ts);
}
}
/**
* 根据指定长度切割集合
*
* @param datas
* @param count
* @param <T>
* @return
*/
public static <T> List<List<T>> splitList(List<T> datas, int count) {
if (isEmpty(datas)) {
return Collections.EMPTY_LIST;
}
List<List<T>> result = new ArrayList<>();
int dataCount = datas.size();
if (dataCount <= count || count == 0) {
result.add(datas);
return result;
}
List<T> tmps = new ArrayList<>();
for (int i = 0; i < dataCount; i++) {
tmps.add(datas.get(i));
if ((i + 1) % count == 0) {
result.add(tmps);
tmps = new ArrayList<>();
}
}
if (isNotEmpty(tmps)) {
result.add(tmps);
}
return result;
}
/**
* 切割set
*
* @param datas
* @param count
* @param <T>
* @return
*/
public static <T> List<Set<T>> splitSet(Set<T> datas, int count) {
if (isEmpty(datas)) {
return Collections.EMPTY_LIST;
}
List<Set<T>> result = new ArrayList<>();
int dataCount = datas.size();
if (dataCount <= count || count == 0) {
result.add(datas);
return result;
}
Set<T> tmps = new HashSet<>();
Iterator<T> iterator = datas.iterator();
int i = 0;
while (iterator.hasNext()) {
tmps.add(iterator.next());
if ((i + 1) % count == 0) {
result.add(tmps);
tmps = new HashSet<>();
}
i++;
}
if (isNotEmpty(tmps)) {
result.add(tmps);
}
return result;
}
public static <T> List<T> subList(List<T> list, int fromIndex, int toIndex) {
if (list != null) {
List<T> subList = new ArrayList<>();
for (int i = fromIndex; i < list.size() && i < toIndex; i++) {
subList.add(list.get(i));
}
return subList;
}
return null;
}
public static void main(String[] args) {
}
}
suggest-task/src/main/java/com/secoo/so/suggest/util/DateUtils.java 0 → 100644
package com.secoo.so.suggest.util;
import java.text.SimpleDateFormat;
import java.util.Date;
public class DateUtils {
public static final String DEFAULT_DATE_FORMAT = "yyyy-MM-dd";
public static final String DEFAULT_DATETIME_FORMAT = "yyyy-MM-dd HH:mm:ss";
public static String currentDate(String format) {
return new SimpleDateFormat(format).format(new Date());
}
public static String currentDate() {
return currentDate(DEFAULT_DATE_FORMAT);
}
public static String currentDatetime(String format) {
return new SimpleDateFormat(format).format(new Date());
}
public static String currentDatetime() {
return currentDatetime(DEFAULT_DATETIME_FORMAT);
}
public static String formatDate(long ms) {
return formatDate(new Date(ms));
}
public static String formatDate(long ms, String format) {
return formatDate(new Date(ms), format);
}
public static String formatDate(Date date) {
return new SimpleDateFormat(DEFAULT_DATETIME_FORMAT).format(date);
}
public static String formatDate(Date date, String format) {
return new SimpleDateFormat(format).format(date);
}
}
suggest-task/src/main/java/com/secoo/so/suggest/util/FeiShuUtil.java 0 → 100644
package com.secoo.so.suggest.util;
import com.alibaba.fastjson.JSON;
import com.secoo.so.suggest.config.ConfigUtil;
import org.apache.http.HttpStatus;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.net.URI;
import java.util.*;
import java.util.concurrent.*;
public class FeiShuUtil {
private static final Logger LOGGER = LoggerFactory.getLogger(FeiShuUtil.class);
private static CloseableHttpClient client = HttpClientBuilder.create().build();
private static final URI FEI_SHU_URL = URI.create("http://matrix-inform.secoolocal.com/user/sendToUser");
/**
* 单线程的线程池发送消息，避免阻塞主线程
*/
private static ExecutorService executor = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS,
new LinkedBlockingQueue<>(1024), Executors.defaultThreadFactory(), new ThreadPoolExecutor.DiscardPolicy());
public static void sendMessage(String title, String message) {
sendMessage(title, message, null);
}
public static void sendMessage(String title, String message, List<String> phones) {
if (StringUtils.isBlank(message)) {
return;
}
if (StringUtils.isBlank(title)) {
title = "异常通知";
}
final Map<String, Object> params = new HashMap<>(16);
params.put("title", title);
params.put("body", Collections.singletonList(message));
params.put("phones", phones);
if(CollectionUtils.isEmpty(phones)){
phones = StringUtils.splitToList(ConfigUtil.getString("suggestTask.warningPhones"), ",");
}
final String fTitle = title;
final String fMessage = message;
final String fPhones = StringUtils.join(phones, ",");
Runnable runnable = new Runnable() {
@Override
public void run() {
CloseableHttpResponse res;
try {
HttpPost post = new HttpPost();
post.setURI(FEI_SHU_URL);
post.setEntity(new StringEntity(JSON.toJSON(params).toString(), ContentType.APPLICATION_JSON));
res = client.execute(post);
if (res == null || res.getEntity() == null) {
LOGGER.error("发送飞书消息失败: title:{}, message:{}, phones:{}", fTitle, fMessage, fPhones);
return;
}
if (res.getStatusLine() == null || res.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
LOGGER.error("发送飞书消息失败,title:{}, message:{}, phones:{}, res:{}", fTitle, fMessage, fPhones, EntityUtils.toString(res.getEntity()));
return;
}
} catch (IOException e) {
LOGGER.error("发送飞书消息失败: title:{}, message:{}, phones:{}", fTitle, fMessage, fPhones, e);
}
}
};
executor.submit(runnable);
}
public static void waitForFinish() throws InterruptedException {
executor.shutdown();
executor.awaitTermination(5, TimeUnit.MINUTES);
}
public static void main(String[] args) throws InterruptedException {
FeiShuUtil.sendMessage("测试", "hello wolrd", Arrays.asList("13426233960"));
FeiShuUtil.waitForFinish();
}
}
\ No newline at end of file
suggest-task/src/main/java/com/secoo/so/suggest/util/FileUtils.java 0 → 100644
package com.secoo.so.suggest.util;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
public class FileUtils {
private static Logger log = LoggerFactory.getLogger(FileUtils.class);
/**
* 创建文件
*
* @param fileName 文件名字
* @return File实例
*/
public static File createFile(String fileName) {
File file = new File(fileName);
if (!file.getParentFile().exists()) {
file.getParentFile().mkdirs();
}
if (file.exists()) {
// throw new RuntimeException("文件已存在：" + fileName);
// 文件存在则删除原有文件，直接覆盖
file.delete();
file = new File(fileName);
}
return file;
}
public static void delFile(File file) {
if (file.exists()) {
if (file.isFile()) {
file.delete();
} else {
File[] files = file.listFiles();
for (File f : files) {
delFile(f);
}
file.delete();
}
}
}
public static void delFile(String path) {
delFile(new File(path));
}
/**
* copy 文件
*
* @param sourceFile
* @param targetFile
* @throws IOException
*/
public static void copyFile(File sourceFile, File targetFile) throws IOException {
saveFile(new FileInputStream(sourceFile), targetFile);
}
public static void saveFile(InputStream inputStream, File targetFile) throws IOException {
BufferedInputStream inBuff = null;
BufferedOutputStream outBuff = null;
try {
// 新建文件输入流并对它进行缓冲
inBuff = new BufferedInputStream(inputStream);
// 新建文件输出流并对它进行缓冲
outBuff = new BufferedOutputStream(new FileOutputStream(targetFile));
// 缓冲数组
byte[] b = new byte[1024 * 5];
int len;
while ((len = inBuff.read(b)) != -1) {
outBuff.write(b, 0, len);
}
// 刷新此缓冲的输出流
outBuff.flush();
} finally {
// 关闭流
if (inBuff != null)
inBuff.close();
if (outBuff != null)
outBuff.close();
}
}
public static void saveFile(InputStream inputStream, String targetFile) throws IOException {
saveFile(inputStream, createFile(targetFile));
}
private void saveToFile(List<String> lines, String fileName) {
saveToFile(lines, fileName, true);
}
private void saveToFile(String content, String fileName) {
saveToFile(content, fileName, true);
}
public static void saveToFile(List<String> lines, String fileName, boolean append) {
saveToFile(lines, new File(fileName), append);
}
public static void saveToFile(String content, String fileName, boolean append) {
saveToFile(content, new File(fileName), append);
}
public static void saveToFile(List<String> lines, File file, boolean append) {
if (CollectionUtils.isNotEmpty(lines)) {
StringBuilder sBuilder = new StringBuilder();
lines.forEach(line -> {
sBuilder.append(line).append("\n");
});
saveToFile(sBuilder.toString(), file, append);
}
}
public static void saveToFile(String content, File file, boolean append) {
FileWriter fw = null;
BufferedWriter bw = null;
try {
if (!file.getParentFile().exists()) {
file.getParentFile().mkdirs();
}
fw = new FileWriter(file, append);
bw = new BufferedWriter(fw);
bw.write(content);
} catch (IOException e) {
log.error(e.getMessage(), e);
} catch (Exception e) {
log.error(e.getMessage(), e);
} finally {
// 完毕，关闭所有链接
try {
bw.close();
fw.close();
} catch (IOException e) {
log.error(e.getMessage(), e);
}
}
}
public static File getFile(String fileName, String propertyName) throws Exception {
String filePath = null;
if (propertyName != null && !"".equals(propertyName)) {
filePath = System.getProperty(propertyName);
}
File file = null;
if (filePath == null || "".equals(filePath)) {
URL url = FileUtils.class.getClassLoader().getResource(propertyName + fileName);
if (url == null) {
throw new FileNotFoundException(fileName + " not found!");
}
file = new File(url.getPath());
} else {
filePath = filePath.endsWith("/") ? filePath.concat(fileName) : filePath.concat("/").concat(fileName);
file = new File(filePath);
}
return file;
}
public static String getFilePath(String fileName, String propertyName) throws Exception {
String filePath = null;
if (propertyName != null && !"".equals(propertyName)) {
filePath = System.getProperty(propertyName);
}
if (filePath == null || "".equals(filePath)) {
URL url = FileUtils.class.getClassLoader().getResource(propertyName + fileName);
if (url == null) {
throw new FileNotFoundException(fileName + " not found!");
}
filePath = url.getPath();
} else {
filePath = filePath.endsWith("/") ? filePath.concat(fileName) : filePath.concat("/").concat(fileName);
}
return filePath;
}
public static String getFileDir(String fileName, String propertyName) throws Exception {
String filePath = null;
if (propertyName != null && !"".equals(propertyName)) {
filePath = System.getProperty(propertyName);
}
if (filePath == null || "".equals(filePath)) {
URL url = FileUtils.class.getClassLoader().getResource(propertyName + fileName);
if (url == null) {
throw new FileNotFoundException(fileName + " not found!");
}
filePath = url.getPath();
filePath = filePath.replace(fileName, "");
} else {
filePath = filePath.endsWith("/") ? filePath.concat(fileName) : filePath.concat("/").concat(fileName);
}
return filePath;
}
/**
* @param file
*/
public static String read(File file, String charset) {
final byte[] content = read(file);
return content == null ? "" : new String(content);
}
public static byte[] read(File file) {
if (!(file.exists() && file.isFile())) {
throw new IllegalArgumentException("The remote not exist or not a remote");
}
FileInputStream fis = null;
byte[] content = null;
try {
fis = new FileInputStream(file);
content = new byte[fis.available()];
fis.read(content);
} catch (FileNotFoundException e) {
log.error(e.getMessage(), e);
} catch (IOException e) {
log.error(e.getMessage(), e);
} finally {
if (fis != null) {
try {
fis.close();
} catch (IOException e) {
log.error(e.getMessage(), e);
}
fis = null;
}
}
return content;
}
/**
* 将saveProperties保存为文件
*
* @param filePath
* @param parameterName
* @param parameterValue
*/
public static void saveProperties(String filePath, String parameterName, String parameterValue) {
Properties prop = new Properties();
try {
InputStream fis = new FileInputStream(filePath);
prop.load(fis);
OutputStream fos = new FileOutputStream(filePath);
prop.setProperty(parameterName, parameterValue);
prop.store(fos, "Update '" + parameterName + "' value");
fis.close();
} catch (IOException e) {
System.err.println("Visit " + filePath + " for updating " + parameterName + " value error");
}
}
/**
* 读取文件
*
* @param inputStream 文件流
* @return
* @author shaoqiang.guo
*/
public static String readFile(InputStream inputStream) {
BufferedInputStream in = new BufferedInputStream(inputStream);
ByteArrayOutputStream out = new ByteArrayOutputStream(1024);
byte[] temp = new byte[1024];
int size = 0;
try {
while ((size = in.read(temp)) != -1) {
out.write(temp, 0, size);
}
} catch (IOException e) {
throw new RuntimeException("read file error.", e);
} finally {
try {
in.close();
} catch (IOException e) {
throw new RuntimeException("close stream error.", e);
}
}
byte[] content = out.toByteArray();
return new String(content);
}
// 通过url下载文件保存到本地
public static void download(String urlString, String fileName) throws Exception {
// 构造URL
URL url = new URL(urlString);
// 打开连接
URLConnection con = url.openConnection();
con.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt)");
con.setRequestProperty("sec-ch-ua", "\"Google Chrome\";v=\"93\", \" Not;A Brand\";v=\"99\", \"Chromium\";v=\"93\"");
con.setRequestProperty("cache-control", "no-cache");
con.setRequestProperty("access-control-expose-headers", "x-ak-country-code");
// 输入流
InputStream is = con.getInputStream();
// 1K的数据缓冲
byte[] bs = new byte[1024];
// 读取到的数据长度
int len;
// 输出的文件流
File file = new File(fileName);
FileOutputStream os = new FileOutputStream(file, true);
// 开始读取
while ((len = is.read(bs)) != -1) {
os.write(bs, 0, len);
}
// 完毕，关闭所有链接
os.close();
is.close();
}
// 通过url下载文件保存到本地
public static void download2(String urlString, String fileName) throws Exception {
// 构造URL
URL url = new URL(urlString);
HttpURLConnection httpURLConnection = (HttpURLConnection) new URL(urlString).openConnection();
httpURLConnection.setRequestMethod("GET");
httpURLConnection.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36");
httpURLConnection.setRequestProperty("Accept-Encoding", "gzip");
httpURLConnection.setRequestProperty("Referer", "no-referrer");
httpURLConnection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
httpURLConnection.setConnectTimeout(15000);
httpURLConnection.setReadTimeout(20000);
// 输入流
InputStream is = httpURLConnection.getInputStream();
// 1K的数据缓冲
byte[] bs = new byte[1024];
// 读取到的数据长度
int len;
// 输出的文件流
File file = new File(fileName);
FileOutputStream os = new FileOutputStream(file, true);
// 开始读取
while ((len = is.read(bs)) != -1) {
os.write(bs, 0, len);
}
// 完毕，关闭所有链接
os.close();
is.close();
}
public static List<String> readLines(File file) {
List<String> lines = new ArrayList<>();
try {
// 1. .csv文件的路径。注意只有一个\的要改成
BufferedReader br = new BufferedReader(new FileReader(file));
String line;
// 读取到的内容给line变量
while ((line = br.readLine()) != null) {
if (StringUtils.isNotBlank(line)) {
lines.add(line);
}
}
} catch (Exception e) {
log.error("readLines from file<{}> error:", e);
}
return lines;
}
public static List<String> readLines(String fileName) {
return readLines(new File(fileName));
}
}
suggest-task/src/main/java/com/secoo/so/suggest/util/ObjectUtils.java 0 → 100644
package com.secoo.so.suggest.util;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.TypeReference;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.beans.BeanInfo;
import java.beans.Introspector;
import java.beans.PropertyDescriptor;
import java.io.*;
import java.lang.reflect.*;
import java.math.BigDecimal;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.Statement;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
/**
* change this template use File | Settings | File Templates.
*/
public final class ObjectUtils {
private static final Logger logger = LoggerFactory.getLogger(ObjectUtils.class);
private static final int INITIAL_HASH = 7;
private static final int MULTIPLIER = 31;
private static final String EMPTY_STRING = "";
private static final String NULL_STRING = "null";
private static final String ARRAY_START = "{";
private static final String ARRAY_END = "}";
private static final String EMPTY_ARRAY = ARRAY_START + ARRAY_END;
private static final String ARRAY_ELEMENT_SEPARATOR = ", ";
@SuppressWarnings("rawtypes")
public static Boolean isWrapClass(Class clazz) {
try {
return ((Class) clazz.getField("TYPE").get(null)).isPrimitive();
} catch (Exception e) {
return false;
}
}
/**
* 对象克隆，支持单个对象，数组与集合
*
* @param src
* @return
*/
public static Object byteClone(Object src) {
try {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
ObjectOutputStream out = new ObjectOutputStream(baos);
out.writeObject(src);
out.close();
ByteArrayInputStream bin = new ByteArrayInputStream(baos.toByteArray());
ObjectInputStream in = new ObjectInputStream(bin);
Object clone = in.readObject();
in.close();
return (clone);
} catch (ClassNotFoundException e) {
throw new InternalError(e.toString());
} catch (StreamCorruptedException e) {
throw new InternalError(e.toString());
} catch (IOException e) {
throw new InternalError(e.toString());
}
}
public static <T> T mapToObject(Map<String, Object> map, Class<T> beanClass) throws Exception {
if (map == null) {
return null;
}
T obj = beanClass.newInstance();
BeanInfo beanInfo = Introspector.getBeanInfo(obj.getClass());
PropertyDescriptor[] propertyDescriptors = beanInfo.getPropertyDescriptors();
for (PropertyDescriptor property : propertyDescriptors) {
Method setter = property.getWriteMethod();
if (setter != null && map.containsKey(property.getName())) {
try {
setter.invoke(obj, map.get(property.getName()));
} catch (Exception e) {
logger.error(property.getName() + " value " + map.get(property.getName()) + " type not match", e);
}
}
}
return obj;
}
public static Map<String, Object> objectToMap(Object obj) throws Exception {
if (obj == null) {
return null;
}
Map<String, Object> map = new HashMap<String, Object>();
BeanInfo beanInfo = Introspector.getBeanInfo(obj.getClass());
PropertyDescriptor[] propertyDescriptors = beanInfo.getPropertyDescriptors();
for (PropertyDescriptor property : propertyDescriptors) {
String key = property.getName();
if (key.compareToIgnoreCase("class") == 0) {
continue;
}
Method getter = property.getReadMethod();
Object value = getter != null ? getter.invoke(obj) : null;
map.put(key, value);
}
return map;
}
public static boolean isEmpty(Object obj) {
return obj == null;
}
@SuppressWarnings("rawtypes")
public static boolean isEmpty(Collection objects) {
return objects == null || objects.size() == 0;
}
/**
* Return whether the given throwable is a checked exception: that is,
* neither a RuntimeException nor an Error.
*
* @param ex the throwable to check
* @return whether the throwable is a checked exception
* @see Exception
* @see RuntimeException
* @see Error
*/
public static boolean isCheckedException(Throwable ex) {
return !(ex instanceof RuntimeException || ex instanceof Error);
}
/**
* Check whether the given exception is compatible with the exceptions
* declared in a throws clause.
*
* @param ex the exception to checked
* @param declaredExceptions the exceptions declared in the throws clause
* @return whether the given exception is compatible
*/
@SuppressWarnings({"rawtypes", "unchecked"})
public static boolean isCompatibleWithThrowsClause(Throwable ex, Class[] declaredExceptions) {
if (!isCheckedException(ex)) {
return true;
}
if (declaredExceptions != null) {
int i = 0;
while (i < declaredExceptions.length) {
if (declaredExceptions[i].isAssignableFrom(ex.getClass())) {
return true;
}
i++;
}
}
return false;
}
/**
* Determine whether the given object is an array: either an Object array or
* a primitive array.
*
* @param obj the object to check
*/
public static boolean isArray(Object obj) {
return (obj != null && obj.getClass().isArray());
}
/**
* Determine whether the given array is empty: i.e. <code>null</code> or of
* zero length.
*
* @param array the array to check
*/
public static boolean isEmpty(Object[] array) {
return (array == null || array.length == 0);
}
/**
* Check whether the given array contains the given element.
*
* @param array the array to check (may be <code>null</code>, in which case
* the return value will always be <code>false</code>)
* @param element the element to check for
* @return whether the element has been found in the given array
*/
public static boolean containsElement(Object[] array, Object element) {
if (array == null) {
return false;
}
for (Object arrayEle : array) {
if (nullSafeEquals(arrayEle, element)) {
return true;
}
}
return false;
}
/**
* Check whether the given array of enum constants contains a constant with
* the given name, ignoring case when determining a match.
*
* @param enumValues the enum values to check, typically the reports of a call to
* MyEnum.values()
* @param constant the constant name to find (must not be null or empty string)
* @return whether the constant has been found in the given array
*/
public static boolean containsConstant(Enum<?>[] enumValues, String constant) {
return containsConstant(enumValues, constant, false);
}
/**
* Check whether the given array of enum constants contains a constant with
* the given name.
*
* @param enumValues the enum values to check, typically the reports of a call to
* MyEnum.values()
* @param constant the constant name to find (must not be null or empty string)
* @param caseSensitive whether case is significant in determining a match
* @return whether the constant has been found in the given array
*/
public static boolean containsConstant(Enum<?>[] enumValues, String constant, boolean caseSensitive) {
for (Enum<?> candidate : enumValues) {
if (caseSensitive ? candidate.toString().equals(constant)
: candidate.toString().equalsIgnoreCase(constant)) {
return true;
}
}
return false;
}
/**
* Case insensitive alternative to {@link Enum#valueOf(Class, String)}.
*
* @param <E> the concrete Enum type
* @param enumValues the array of all Enum constants in question, usually per
* Enum.values()
* @param constant the constant to get the enum value of
* @throws IllegalArgumentException if the given constant is not found in the given array of enum
* values. Use {@link #containsConstant(Enum[], String)} as a
* guard to avoid this exception.
*/
public static <E extends Enum<?>> E caseInsensitiveValueOf(E[] enumValues, String constant) {
for (E candidate : enumValues) {
if (candidate.toString().equalsIgnoreCase(constant)) {
return candidate;
}
}
throw new IllegalArgumentException(String.format("constant [%s] does not exist in enum type %s", constant,
enumValues.getClass().getComponentType().getName()));
}
/**
* Append the given object to the given array, returning a new array
* consisting of the input array contents plus the given object.
*
* @param array the array to append to (can be <code>null</code>)
* @param obj the object to append
* @return the new array (of the same component type; never
* <code>null</code>)
*/
public static <A, O extends A> A[] addObjectToArray(A[] array, O obj) {
Class<?> compType = Object.class;
if (array != null) {
compType = array.getClass().getComponentType();
} else if (obj != null) {
compType = obj.getClass();
}
int newArrLength = (array != null ? array.length + 1 : 1);
@SuppressWarnings("unchecked")
A[] newArr = (A[]) Array.newInstance(compType, newArrLength);
if (array != null) {
System.arraycopy(array, 0, newArr, 0, array.length);
}
newArr[newArr.length - 1] = obj;
return newArr;
}
/**
* Convert the given array (which may be a primitive array) to an object
* array (if necessary of primitive wrapper objects).
* <p>
* A <code>null</code> source value will be converted to an empty Object
* array.
*
* @param source the (potentially primitive) array
* @return the corresponding object array (never <code>null</code>)
* @throws IllegalArgumentException if the parameter is not an array
*/
@SuppressWarnings("rawtypes")
public static Object[] toObjectArray(Object source) {
if (source instanceof Object[]) {
return (Object[]) source;
}
if (source == null) {
return new Object[0];
}
if (!source.getClass().isArray()) {
throw new IllegalArgumentException("Source is not an array: " + source);
}
int length = Array.getLength(source);
if (length == 0) {
return new Object[0];
}
Class wrapperType = Array.get(source, 0).getClass();
Object[] newArray = (Object[]) Array.newInstance(wrapperType, length);
for (int i = 0; i < length; i++) {
newArray[i] = Array.get(source, i);
}
return newArray;
}
// ---------------------------------------------------------------------
// Convenience methods for content-based equality/hash-code handling
// ---------------------------------------------------------------------
/**
* Determine if the given objects are equal, returning <code>true</code> if
* both are <code>null</code> or <code>false</code> if only one is
* <code>null</code>.
* <p>
* Compares arrays with <code>Arrays.equals</code>, performing an equality
* check based on the array elements rather than the array reference.
*
* @param o1 first Object to compare
* @param o2 second Object to compare
* @return whether the given objects are equal
* @see Arrays#equals
*/
public static boolean nullSafeEquals(Object o1, Object o2) {
if (o1 == o2) {
return true;
}
if (o1 == null || o2 == null) {
return false;
}
if (o1.equals(o2)) {
return true;
}
if (o1.getClass().isArray() && o2.getClass().isArray()) {
if (o1 instanceof Object[] && o2 instanceof Object[]) {
return Arrays.equals((Object[]) o1, (Object[]) o2);
}
if (o1 instanceof boolean[] && o2 instanceof boolean[]) {
return Arrays.equals((boolean[]) o1, (boolean[]) o2);
}
if (o1 instanceof byte[] && o2 instanceof byte[]) {
return Arrays.equals((byte[]) o1, (byte[]) o2);
}
if (o1 instanceof char[] && o2 instanceof char[]) {
return Arrays.equals((char[]) o1, (char[]) o2);
}
if (o1 instanceof double[] && o2 instanceof double[]) {
return Arrays.equals((double[]) o1, (double[]) o2);
}
if (o1 instanceof float[] && o2 instanceof float[]) {
return Arrays.equals((float[]) o1, (float[]) o2);
}
if (o1 instanceof int[] && o2 instanceof int[]) {
return Arrays.equals((int[]) o1, (int[]) o2);
}
if (o1 instanceof long[] && o2 instanceof long[]) {
return Arrays.equals((long[]) o1, (long[]) o2);
}
if (o1 instanceof short[] && o2 instanceof short[]) {
return Arrays.equals((short[]) o1, (short[]) o2);
}
}
return false;
}
/**
* Return as hash code for the given object; typically the value of
* <code>{@link Object#hashCode()}</code>. If the object is an array, this
* method will delegate to any of the <code>nullSafeHashCode</code> methods
* for arrays in this class. If the object is <code>null</code>, this method
* returns 0.
*
* @see #nullSafeHashCode(Object[])
* @see #nullSafeHashCode(boolean[])
* @see #nullSafeHashCode(byte[])
* @see #nullSafeHashCode(char[])
* @see #nullSafeHashCode(double[])
* @see #nullSafeHashCode(float[])
* @see #nullSafeHashCode(int[])
* @see #nullSafeHashCode(long[])
* @see #nullSafeHashCode(short[])
*/
public static int nullSafeHashCode(Object obj) {
if (obj == null) {
return 0;
}
if (obj.getClass().isArray()) {
if (obj instanceof Object[]) {
return nullSafeHashCode((Object[]) obj);
}
if (obj instanceof boolean[]) {
return nullSafeHashCode((boolean[]) obj);
}
if (obj instanceof byte[]) {
return nullSafeHashCode((byte[]) obj);
}
if (obj instanceof char[]) {
return nullSafeHashCode((char[]) obj);
}
if (obj instanceof double[]) {
return nullSafeHashCode((double[]) obj);
}
if (obj instanceof float[]) {
return nullSafeHashCode((float[]) obj);
}
if (obj instanceof int[]) {
return nullSafeHashCode((int[]) obj);
}
if (obj instanceof long[]) {
return nullSafeHashCode((long[]) obj);
}
if (obj instanceof short[]) {
return nullSafeHashCode((short[]) obj);
}
}
return obj.hashCode();
}
/**
* Return a hash code based on the contents of the specified array. If
* <code>array</code> is <code>null</code>, this method returns 0.
*/
public static int nullSafeHashCode(Object[] array) {
if (array == null) {
return 0;
}
int hash = INITIAL_HASH;
int arraySize = array.length;
for (int i = 0; i < arraySize; i++) {
hash = MULTIPLIER * hash + nullSafeHashCode(array[i]);
}
return hash;
}
/**
* Return a hash code based on the contents of the specified array. If
* <code>array</code> is <code>null</code>, this method returns 0.
*/
public static int nullSafeHashCode(boolean[] array) {
if (array == null) {
return 0;
}
int hash = INITIAL_HASH;
int arraySize = array.length;
for (int i = 0; i < arraySize; i++) {
hash = MULTIPLIER * hash + hashCode(array[i]);
}
return hash;
}
/**
* Return a hash code based on the contents of the specified array. If
* <code>array</code> is <code>null</code>, this method returns 0.
*/
public static int nullSafeHashCode(byte[] array) {
if (array == null) {
return 0;
}
int hash = INITIAL_HASH;
int arraySize = array.length;
for (int i = 0; i < arraySize; i++) {
hash = MULTIPLIER * hash + array[i];
}
return hash;
}
/**
* Return a hash code based on the contents of the specified array. If
* <code>array</code> is <code>null</code>, this method returns 0.
*/
public static int nullSafeHashCode(char[] array) {
if (array == null) {
return 0;
}
int hash = INITIAL_HASH;
int arraySize = array.length;
for (int i = 0; i < arraySize; i++) {
hash = MULTIPLIER * hash + array[i];
}
return hash;
}
/**
* Return a hash code based on the contents of the specified array. If
* <code>array</code> is <code>null</code>, this method returns 0.
*/
@SuppressWarnings("unused")
public static int nullSafeHashCode(double[] array) {
if (array == null) {
return 0;
}
int hash = INITIAL_HASH;
int arraySize = array.length;
for (double anArray : array) {
hash = MULTIPLIER * hash + hashCode(anArray);
}
return hash;
}
/**
* Return a hash code based on the contents of the specified array. If
* <code>array</code> is <code>null</code>, this method returns 0.
*/
@SuppressWarnings("unused")
public static int nullSafeHashCode(float[] array) {
if (array == null) {
return 0;
}
int hash = INITIAL_HASH;
int arraySize = array.length;
for (float anArray : array) {
hash = MULTIPLIER * hash + hashCode(anArray);
}
return hash;
}
/**
* Return a hash code based on the contents of the specified array. If
* <code>array</code> is <code>null</code>, this method returns 0.
*/
@SuppressWarnings("unused")
public static int nullSafeHashCode(int[] array) {
if (array == null) {
return 0;
}
int hash = INITIAL_HASH;
int arraySize = array.length;
for (int anArray : array) {
hash = MULTIPLIER * hash + anArray;
}
return hash;
}
/**
* Return a hash code based on the contents of the specified array. If
* <code>array</code> is <code>null</code>, this method returns 0.
*/
@SuppressWarnings("unused")
public static int nullSafeHashCode(long[] array) {
if (array == null) {
return 0;
}
int hash = INITIAL_HASH;
int arraySize = array.length;
for (long anArray : array) {
hash = MULTIPLIER * hash + hashCode(anArray);
}
return hash;
}
/**
* Return a hash code based on the contents of the specified array. If
* <code>array</code> is <code>null</code>, this method returns 0.
*/
@SuppressWarnings("unused")
public static int nullSafeHashCode(short[] array) {
if (array == null) {
return 0;
}
int hash = INITIAL_HASH;
int arraySize = array.length;
for (short anArray : array) {
hash = MULTIPLIER * hash + anArray;
}
return hash;
}
/**
* Return the same value as <code>{@link Boolean#hashCode()}</code>.
*
* @see Boolean#hashCode()
*/
public static int hashCode(boolean bool) {
return bool ? 1231 : 1237;
}
/**
* Return the same value as <code>{@link Double#hashCode()}</code>.
*
* @see Double#hashCode()
*/
public static int hashCode(double dbl) {
long bits = Double.doubleToLongBits(dbl);
return hashCode(bits);
}
/**
* Return the same value as <code>{@link Float#hashCode()}</code>.
*
* @see Float#hashCode()
*/
public static int hashCode(float flt) {
return Float.floatToIntBits(flt);
}
/**
* Return the same value as <code>{@link Long#hashCode()}</code>.
*
* @see Long#hashCode()
*/
public static int hashCode(long lng) {
return (int) (lng ^ (lng >>> 32));
}
// ---------------------------------------------------------------------
// Convenience methods for toString output
// ---------------------------------------------------------------------
/**
* Return a String representation of an object's overall identity.
*
* @param obj the object (may be <code>null</code>)
* @return the object's identity as String representation, or an empty
* String if the object was <code>null</code>
*/
public static String identityToString(Object obj) {
if (obj == null) {
return EMPTY_STRING;
}
return obj.getClass().getName() + "@" + getIdentityHexString(obj);
}
/**
* Return a hex String form of an object's identity hash code.
*
* @param obj the object
* @return the object's identity code in hex notation
*/
public static String getIdentityHexString(Object obj) {
return Integer.toHexString(System.identityHashCode(obj));
}
/**
* Return a content-based String representation if <code>obj</code> is not
* <code>null</code>; otherwise returns an empty String.
* <p>
* Differs from {@link #nullSafeToString(Object)} in that it returns an
* empty String rather than "null" for a <code>null</code> value.
*
* @param obj the object to build a display String for
* @return a display String representation of <code>obj</code>
* @see #nullSafeToString(Object)
*/
public static String getDisplayString(Object obj) {
if (obj == null) {
return EMPTY_STRING;
}
return nullSafeToString(obj);
}
/**
* Determine the class name for the given object.
* <p>
* Returns <code>"null"</code> if <code>obj</code> is <code>null</code>.
*
* @param obj the object to introspect (may be <code>null</code>)
* @return the corresponding class name
*/
public static String nullSafeClassName(Object obj) {
return (obj != null ? obj.getClass().getName() : NULL_STRING);
}
/**
* Return a String representation of the specified Object.
* <p>
* Builds a String representation of the contents in case of an array.
* Returns <code>"null"</code> if <code>obj</code> is <code>null</code>.
*
* @param obj the object to build a String representation for
* @return a String representation of <code>obj</code>
*/
public static String nullSafeToString(Object obj) {
if (obj == null) {
return NULL_STRING;
}
if (obj instanceof String) {
return (String) obj;
}
if (obj instanceof Object[]) {
return nullSafeToString((Object[]) obj);
}
if (obj instanceof boolean[]) {
return nullSafeToString((boolean[]) obj);
}
if (obj instanceof byte[]) {
return nullSafeToString((byte[]) obj);
}
if (obj instanceof char[]) {
return nullSafeToString((char[]) obj);
}
if (obj instanceof double[]) {
return nullSafeToString((double[]) obj);
}
if (obj instanceof float[]) {
return nullSafeToString((float[]) obj);
}
if (obj instanceof int[]) {
return nullSafeToString((int[]) obj);
}
if (obj instanceof long[]) {
return nullSafeToString((long[]) obj);
}
if (obj instanceof short[]) {
return nullSafeToString((short[]) obj);
}
String str = obj.toString();
return (str != null ? str : EMPTY_STRING);
}
/**
* Return a String representation of the contents of the specified array.
* <p>
* The String representation consists of a list of the array's elements,
* enclosed in curly braces (<code>"{}"</code>). Adjacent elements are
* separated by the characters <code>", "</code> (a comma followed by a
* space). Returns <code>"null"</code> if <code>array</code> is
* <code>null</code>.
*
* @param array the array to build a String representation for
* @return a String representation of <code>array</code>
*/
public static String nullSafeToString(Object[] array) {
if (array == null) {
return NULL_STRING;
}
int length = array.length;
if (length == 0) {
return EMPTY_ARRAY;
}
StringBuilder sb = new StringBuilder();
for (int i = 0; i < length; i++) {
if (i == 0) {
sb.append(ARRAY_START);
} else {
sb.append(ARRAY_ELEMENT_SEPARATOR);
}
sb.append(array[i]);
}
sb.append(ARRAY_END);
return sb.toString();
}
/**
* Return a String representation of the contents of the specified array.
* <p>
* The String representation consists of a list of the array's elements,
* enclosed in curly braces (<code>"{}"</code>). Adjacent elements are
* separated by the characters <code>", "</code> (a comma followed by a
* space). Returns <code>"null"</code> if <code>array</code> is
* <code>null</code>.
*
* @param array the array to build a String representation for
* @return a String representation of <code>array</code>
*/
public static String nullSafeToString(boolean[] array) {
if (array == null) {
return NULL_STRING;
}
int length = array.length;
if (length == 0) {
return EMPTY_ARRAY;
}
StringBuilder sb = new StringBuilder();
for (int i = 0; i < length; i++) {
if (i == 0) {
sb.append(ARRAY_START);
} else {
sb.append(ARRAY_ELEMENT_SEPARATOR);
}
sb.append(array[i]);
}
sb.append(ARRAY_END);
return sb.toString();
}
/**
* Return a String representation of the contents of the specified array.
* <p>
* The String representation consists of a list of the array's elements,
* enclosed in curly braces (<code>"{}"</code>). Adjacent elements are
* separated by the characters <code>", "</code> (a comma followed by a
* space). Returns <code>"null"</code> if <code>array</code> is
* <code>null</code>.
*
* @param array the array to build a String representation for
* @return a String representation of <code>array</code>
*/
public static String nullSafeToString(byte[] array) {
if (array == null) {
return NULL_STRING;
}
int length = array.length;
if (length == 0) {
return EMPTY_ARRAY;
}
StringBuilder sb = new StringBuilder();
for (int i = 0; i < length; i++) {
if (i == 0) {
sb.append(ARRAY_START);
} else {
sb.append(ARRAY_ELEMENT_SEPARATOR);
}
sb.append(array[i]);
}
sb.append(ARRAY_END);
return sb.toString();
}
/**
* Return a String representation of the contents of the specified array.
* <p>
* The String representation consists of a list of the array's elements,
* enclosed in curly braces (<code>"{}"</code>). Adjacent elements are
* separated by the characters <code>", "</code> (a comma followed by a
* space). Returns <code>"null"</code> if <code>array</code> is
* <code>null</code>.
*
* @param array the array to build a String representation for
* @return a String representation of <code>array</code>
*/
public static String nullSafeToString(char[] array) {
if (array == null) {
return NULL_STRING;
}
int length = array.length;
if (length == 0) {
return EMPTY_ARRAY;
}
StringBuilder sb = new StringBuilder();
for (int i = 0; i < length; i++) {
if (i == 0) {
sb.append(ARRAY_START);
} else {
sb.append(ARRAY_ELEMENT_SEPARATOR);
}
sb.append("'").append(array[i]).append("'");
}
sb.append(ARRAY_END);
return sb.toString();
}
/**
* Return a String representation of the contents of the specified array.
* <p>
* The String representation consists of a list of the array's elements,
* enclosed in curly braces (<code>"{}"</code>). Adjacent elements are
* separated by the characters <code>", "</code> (a comma followed by a
* space). Returns <code>"null"</code> if <code>array</code> is
* <code>null</code>.
*
* @param array the array to build a String representation for
* @return a String representation of <code>array</code>
*/
public static String nullSafeToString(double[] array) {
if (array == null) {
return NULL_STRING;
}
int length = array.length;
if (length == 0) {
return EMPTY_ARRAY;
}
StringBuilder sb = new StringBuilder();
for (int i = 0; i < length; i++) {
if (i == 0) {
sb.append(ARRAY_START);
} else {
sb.append(ARRAY_ELEMENT_SEPARATOR);
}
sb.append(array[i]);
}
sb.append(ARRAY_END);
return sb.toString();
}
/**
* Return a String representation of the contents of the specified array.
* <p>
* The String representation consists of a list of the array's elements,
* enclosed in curly braces (<code>"{}"</code>). Adjacent elements are
* separated by the characters <code>", "</code> (a comma followed by a
* space). Returns <code>"null"</code> if <code>array</code> is
* <code>null</code>.
*
* @param array the array to build a String representation for
* @return a String representation of <code>array</code>
*/
public static String nullSafeToString(float[] array) {
if (array == null) {
return NULL_STRING;
}
int length = array.length;
if (length == 0) {
return EMPTY_ARRAY;
}
StringBuilder sb = new StringBuilder();
for (int i = 0; i < length; i++) {
if (i == 0) {
sb.append(ARRAY_START);
} else {
sb.append(ARRAY_ELEMENT_SEPARATOR);
}
sb.append(array[i]);
}
sb.append(ARRAY_END);
return sb.toString();
}
/**
* Return a String representation of the contents of the specified array.
* <p>
* The String representation consists of a list of the array's elements,
* enclosed in curly braces (<code>"{}"</code>). Adjacent elements are
* separated by the characters <code>", "</code> (a comma followed by a
* space). Returns <code>"null"</code> if <code>array</code> is
* <code>null</code>.
*
* @param array the array to build a String representation for
* @return a String representation of <code>array</code>
*/
public static String nullSafeToString(int[] array) {
if (array == null) {
return NULL_STRING;
}
int length = array.length;
if (length == 0) {
return EMPTY_ARRAY;
}
StringBuilder sb = new StringBuilder();
for (int i = 0; i < length; i++) {
if (i == 0) {
sb.append(ARRAY_START);
} else {
sb.append(ARRAY_ELEMENT_SEPARATOR);
}
sb.append(array[i]);
}
sb.append(ARRAY_END);
return sb.toString();
}
/**
* Return a String representation of the contents of the specified array.
* <p>
* The String representation consists of a list of the array's elements,
* enclosed in curly braces (<code>"{}"</code>). Adjacent elements are
* separated by the characters <code>", "</code> (a comma followed by a
* space). Returns <code>"null"</code> if <code>array</code> is
* <code>null</code>.
*
* @param array the array to build a String representation for
* @return a String representation of <code>array</code>
*/
public static String nullSafeToString(long[] array) {
if (array == null) {
return NULL_STRING;
}
int length = array.length;
if (length == 0) {
return EMPTY_ARRAY;
}
StringBuilder sb = new StringBuilder();
for (int i = 0; i < length; i++) {
if (i == 0) {
sb.append(ARRAY_START);
} else {
sb.append(ARRAY_ELEMENT_SEPARATOR);
}
sb.append(array[i]);
}
sb.append(ARRAY_END);
return sb.toString();
}
/**
* Return a String representation of the contents of the specified array.
* <p>
* The String representation consists of a list of the array's elements,
* enclosed in curly braces (<code>"{}"</code>). Adjacent elements are
* separated by the characters <code>", "</code> (a comma followed by a
* space). Returns <code>"null"</code> if <code>array</code> is
* <code>null</code>.
*
* @param array the array to build a String representation for
* @return a String representation of <code>array</code>
*/
public static String nullSafeToString(short[] array) {
if (array == null) {
return NULL_STRING;
}
int length = array.length;
if (length == 0) {
return EMPTY_ARRAY;
}
StringBuilder sb = new StringBuilder();
for (int i = 0; i < length; i++) {
if (i == 0) {
sb.append(ARRAY_START);
} else {
sb.append(ARRAY_ELEMENT_SEPARATOR);
}
sb.append(array[i]);
}
sb.append(ARRAY_END);
return sb.toString();
}
/**
* 公共的销毁对象方法，用于内存回收
*/
public static void destroy(Object obj) {
obj = null;
}
/**
* 安全关闭流
*/
public static void safeClose(Object... objects) {
boolean debugEnabled = logger.isDebugEnabled();
if (objects == null || objects.length == 0) {
logger.info("safeClose(...) was invoked with null or empty array: {}", objects);
return;
}
for (Object obj : objects) {
if (obj != null) {
if (debugEnabled) {
logger.debug("Trying to safely close {}", obj);
}
if (obj instanceof Flushable) {
try {
((Flushable) obj).flush();
} catch (Exception e) {
if (debugEnabled) {
logger.debug("Flushing Flushable failed", e);
}
}
}
if (obj instanceof Closeable) {
try {
((Closeable) obj).close();
} catch (IOException e) {
if (debugEnabled) {
logger.debug("Closing Closeable failed", e);
}
}
} else if (obj instanceof Connection) {
try {
((Connection) obj).close();
} catch (Exception e) {
if (debugEnabled) {
logger.debug("Closing Connection failed", e);
}
}
} else if (obj instanceof Statement) {
try {
((Statement) obj).close();
} catch (Exception e) {
if (debugEnabled) {
logger.debug("Closing Statement failed", e);
}
}
} else if (obj instanceof ResultSet) {
try {
((ResultSet) obj).close();
} catch (Exception e) {
if (debugEnabled) {
logger.debug("Closing ResultSet failed", e);
}
}
} else {
logger.info("obj was neither Closeable, Connection, Statement or ResultSet.");
try {
Method method = obj.getClass().getMethod("close");
if (method == null) {
logger.info("obj did not have a close() method, ignoring");
} else {
method.setAccessible(true);
method.invoke(obj);
}
} catch (InvocationTargetException e) {
logger.warn("Invoking close() by reflection threw exception", e);
} catch (Exception e) {
logger.warn("Could not invoke close() by reflection", e);
}
}
}
}
}
@SuppressWarnings("rawtypes")
public static List<Field> getDeclaredFieldsIncludeSupper(Class clazz) {
List<Field> fieldList = new ArrayList<>();
Class tempClass = clazz;
while (tempClass != null) {// 当父类为null的时候说明到达了最上层的父类(Object类).
fieldList.addAll(Arrays.asList(tempClass.getDeclaredFields()));
tempClass = tempClass.getSuperclass(); // 得到父类,然后赋给自己
}
return fieldList.stream().filter(e -> !Modifier.isStatic(e.getModifiers()))
.collect(Collectors.toList());
}
@SuppressWarnings("rawtypes")
public static Field getDeclaredField(Class clazz, String fieldName) throws NoSuchFieldException {
return getDeclaredField(clazz, fieldName, true);
}
@SuppressWarnings("rawtypes")
public static Field getDeclaredField(Class clazz, String fieldName, boolean throwException)
throws NoSuchFieldException {
Field field = null;
try {
field = clazz.getDeclaredField(fieldName);
} catch (NoSuchFieldException e) {
if (throwException) {
throw new NoSuchFieldException(fieldName + " of " + clazz.getName());
}
}
return field;
}
@SuppressWarnings("rawtypes")
public static Field getDeclaredFieldIncludeSupper(Class clazz, String fieldName) throws NoSuchFieldException {
Field field = null;
field = getDeclaredField(clazz, fieldName, false);
Class parentClass = clazz.getSuperclass(); // 得到父类,然后赋给自己
while (field == null && parentClass != null) {
field = getDeclaredField(parentClass, fieldName, false);
parentClass = parentClass.getSuperclass();
}
if (field == null) {
throw new NoSuchFieldException(fieldName + " of " + clazz.getName());
}
return field;
}
public static <T> Object getFieldValue(T t, Field field) {
Object value = null;
if (field != null) {
try {
boolean fieldAccessible = field.isAccessible();
field.setAccessible(true);// 打开javabean的访问权限
value = field.get(t);
field.setAccessible(fieldAccessible);// 恢复字段访问权限
} catch (IllegalArgumentException | IllegalAccessException e) {
logger.warn("get field<{}> of object<{}> error", field.getName(), JSON.toJSONString(t), e);
}
}
return value;
}
public static <T> void setFieldValue(T t, Class<T> clazz, String fieldName, Object fieldValue) {
Field field;
try {
field = getDeclaredFieldIncludeSupper(clazz, fieldName);
setFieldValue(t, clazz, field, fieldValue);
} catch (Exception e) {
logger.warn("set field<{}> value<{}> error", fieldName, JSON.toJSONString(fieldValue), e);
}
}
public static <T> void setFieldValue(T t, Class<T> clazz, Field field, Object fieldValue) {
if (field == null || fieldValue == null) {
return;
}
try {
boolean fieldAccessible = field.isAccessible();
field.setAccessible(true);// 打开javabean的访问权限
field.set(t, fieldValue);
field.setAccessible(fieldAccessible);// 恢复字段访问权限
} catch (Exception e) {
logger.warn("set field<{}> value<{}> error", field.getName(), JSON.toJSONString(fieldValue), e);
}
}
/**
* 根据数据类型设置字段值<br>
* TODO： 目前只支持部分数据类型
*
* @date: 2018-12-05
* @author xupeng
*/
public static <T> void setFieldValueByDataType(T t, Class<T> clazz, String fieldName, String fieldValue) {
if (StringUtils.isBlank(fieldName) || fieldValue == null) {
return;
}
Field field = null;
try {
field = getDeclaredFieldIncludeSupper(clazz, fieldName);
setFieldValueByDataType(t, clazz, field, fieldValue);
} catch (Exception e) {
logger.warn("set field<{}> value<{}> error", fieldName, JSON.toJSONString(fieldValue), e);
}
}
/**
* 根据数据类型设置字段值<br>
* TODO： 目前只支持部分数据类型
*
* @date: 2018-12-05
* @author xupeng
*/
public static <T> void setFieldValueByDataType(T t, Class<T> clazz, Field field, String fieldValue) {
if (field == null || fieldValue == null) {
return;
}
try {
boolean fieldAccessible = field.isAccessible();
field.setAccessible(true);// 打开javabean的访问权限
String dataType = field.getType().getName();
// 不是字符串类型的话先trim
if (!dataType.equals(String.class.getName())) {
fieldValue = fieldValue.trim();
}
// 根据类型设置属性值
if (dataType.equals(String.class.getName())) {
field.set(t, fieldValue);
} else if (dataType.equals(Integer.class.getName())) {
field.set(t, StringUtils.isNotBlank(fieldValue) ? Integer.valueOf(fieldValue) : null);
} else if (dataType.equals(Long.class.getName())) {
field.set(t, StringUtils.isNotBlank(fieldValue) ? Long.valueOf(fieldValue) : null);
} else if (dataType.equals(Boolean.class.getName())) {
field.set(t, StringUtils.isNotBlank(fieldValue) ? Boolean.valueOf(fieldValue) : null);
} else if (dataType.equals(Double.class.getName())) {
field.set(t, StringUtils.isNotBlank(fieldValue) ? Double.valueOf(fieldValue) : null);
} else if (dataType.equals(Float.class.getName())) {
field.set(t, StringUtils.isNotBlank(fieldValue) ? Float.valueOf(fieldValue) : null);
} else if (dataType.equals(BigDecimal.class.getName())) {
field.set(t, StringUtils.isNotBlank(fieldValue) ? new BigDecimal(fieldValue) : null);
} else if (dataType.equals(BigDecimal.class.getName())) {
field.set(t, StringUtils.isNotBlank(fieldValue) ? new BigDecimal(fieldValue) : null);
} else {
field.set(t, fieldValue);
}
field.setAccessible(fieldAccessible);// 恢复字段访问权限
} catch (Exception e) {
logger.warn("set field<{}> value<{}> error", field.getName(), JSON.toJSONString(fieldValue), e);
}
}
/**
* 将json字符串转换为绝对路径key的map,不支持数组
*
* @date: 2018-12-07
* @author xupeng
*/
public static Map<String, String> parseJsonStrToAbsolutePathKeyMap(String json) {
Map<String, String> map = new HashMap<>();
if (StringUtils.isNotBlank(json)) {
try {
JSONObject jsonObj = JSON.parseObject(json);
map = parseJsonObjectToAbsolutePathKeyMap("", jsonObj, map);
} catch (Exception e) {
logger.error("parseJsonStrToAbsolutePathKeyMap error", e);
}
}
return map;
}
/**
* 将jsonObject转换为绝对路径key的map，不支持数组
*
* @date: 2018-12-07
* @author xupeng
*/
public static Map<String, String> parseJsonObjectToAbsolutePathKeyMap(String preKey, JSONObject jsonObj,
Map<String, String> map) {
if (jsonObj != null && !jsonObj.isEmpty()) {
try {
for (String key : jsonObj.keySet()) {
Object valueObj = jsonObj.get(key);
String currentKey = key;
if (StringUtils.isNotBlank(preKey)) {
currentKey = preKey + "." + key;
}
if (valueObj instanceof JSONObject) {
parseJsonObjectToAbsolutePathKeyMap(currentKey, (JSONObject) valueObj, map);
} else {
map.put(currentKey, jsonObj.getString(key));
}
}
} catch (Exception e) {
logger.error("parseJsonObjectToAbsolutePathKeyMap error", e);
}
}
return map;
}
/**
* Map转换成以叶子节点为key，叶子节点对应的绝对路径为value
* <p>
* 要使用该方法，必须要保证map中所有层级的key都不会重复
*
* @param preKey
* @param source
* @param pathMap
*/
public static void parseMapLeafNodeWithAbsolutePathKeyMap(String preKey, Map<String, Object> source,
Map<String, String> pathMap) {
if (CollectionUtils.isEmpty(source)) {
return;
}
for (Map.Entry<String, Object> s : source.entrySet()) {
Object v = s.getValue();
String currentKey = s.getKey();
String absoluteKey = null;
if (StringUtils.isNotBlank(preKey)) {
absoluteKey = preKey + "." + currentKey;
} else {
absoluteKey = currentKey;
}
if (v instanceof Map) {
parseMapLeafNodeWithAbsolutePathKeyMap(currentKey, (Map<String, Object>) v, pathMap);
} else {
pathMap.put(currentKey, absoluteKey);
}
}
}
/**
* 把Map的所有叶子节点key设置给leafNodeKeys
* 支持多层级的Map，但不支持List
*
* @param leafNodeKeys
* @param source
* @param levelLimit 层级控制 -1为不限制
*/
public static void parseMapLeafNodeKeys(Set<String> leafNodeKeys, Map<String, Object> source, int levelLimit) {
if (CollectionUtils.isEmpty(source)) {
return;
}
for (Map.Entry<String, Object> s : source.entrySet()) {
Object v = s.getValue();
if (v instanceof Map && levelLimit != 0) {
parseMapLeafNodeKeys(leafNodeKeys, (Map<String, Object>) v, levelLimit - 1);
} else {
leafNodeKeys.add(s.getKey());
}
}
}
/**
* 以叶子节点去Map中找到对应的值
*
* @param source
* @param leafNodeKey
* @param levelLimit 控制取数的层级
* @return
*/
public static Object findValueWithLeafNodeKey(Map<String, Object> source, String leafNodeKey, int levelLimit) {
if (CollectionUtils.isEmpty(source)) {
return null;
}
//先直接查看一下本层级是否存在叶子节点的值
if (source.containsKey(leafNodeKey)) {
Object value = source.get(leafNodeKey);
if (value instanceof Map || value instanceof List) {
return JSON.toJSONString(value);
} else {
return value;
}
}
if (levelLimit == 0) {
return null;
}
for (Map.Entry<String, Object> s : source.entrySet()) {
Object v = s.getValue();
//只对map进一步进行处理
if (v instanceof Map) {
Object value = findValueWithLeafNodeKey((Map<String, Object>) v, leafNodeKey, levelLimit - 1);
if (null == value) {
continue;
} else {
return value;
}
}
}
return null;
}
/**
* 根据路径去找到map中的对应数据
*
* @param source
* @param path
* @param type
* @param <T>
* @return
*/
public static <T> T findValueWithPath(Map<String, Object> source, String path, Class<T> type) {
if (CollectionUtils.isEmpty(source) || StringUtils.isBlank(path)) {
return null;
}
int firstIndex = path.indexOf(".");
if (firstIndex == -1) {
return findValueWithKey(source, path, type);
}
String firstKey = path.substring(0, firstIndex);
String lastKey = path.substring(firstIndex + 1, path.length());
Object nextData = source.get(firstKey);
if (null == nextData || !(nextData instanceof Map)) {
return null;
}
return findValueWithPath((Map<String, Object>) nextData, lastKey, type);
}
public static <T> T findValueWithKey(Map<String, Object> source, String key, Class<T> type) {
Object data = source.get(key);
if (null == data) {
return null;
}
return type.cast(data);
}
public static boolean sleep(Number timeout, TimeUnit timeUnit) {
try {
timeUnit.sleep(timeout.longValue());
return true;
} catch (InterruptedException var3) {
return false;
}
}
public static boolean sleep(Number millis) {
if (millis == null) {
return true;
} else {
try {
Thread.sleep(millis.longValue());
return true;
} catch (InterruptedException var2) {
return false;
}
}
}
public static boolean safeSleep(Number millis) {
long millisLong = millis.longValue();
long before;
long after;
for (long done = 0L; done < millisLong; done += after - before) {
before = System.currentTimeMillis();
if (!sleep(millisLong - done)) {
return false;
}
after = System.currentTimeMillis();
}
return true;
}
/**
* 集合Map转具体对象集合
*
* @param sourceList
* @param clazz
* @param <T>
* @return
*/
public static <T> List<T> listToObjects(List<Map<String, Object>> sourceList, Class<T> clazz) {
if (CollectionUtils.isEmpty(sourceList)) {
return new ArrayList<>();
}
try {
String json = JSON.toJSONString(sourceList);
return JSON.parseArray(json, clazz);
} catch (Exception e) {
logger.error(e.getMessage(), e);
return new ArrayList<>();
}
}
public static <T> T mapToObjectWithJSON(Map<String, Object> sourceMap, Class<T> calzz) {
if (CollectionUtils.isEmpty(sourceMap)) {
return null;
}
try {
String json = JSON.toJSONString(sourceMap);
return JSON.parseObject(json, calzz);
} catch (Exception e) {
logger.error(e.getMessage(), e);
return null;
}
}
public static void main(String[] args) {
/*String json = "{\"name\":\"TMev344\",\"cluster_name\":\"elasticsearch\",\"cluster_uuid\":\"cRwLaMCuRkmZvpnYrcQ6mQ\",\"version\":{\"number\":\"6.5.0\",\"build_flavor\":\"default\",\"build_type\":\"zip\",\"build_hash\":\"816e6f6\",\"build_date\":\"2018-11-09T18:58:36.352602Z\",\"build_snapshot\":false,\"lucene_version\":\"7.5.0\",\"minimum_wire_compatibility_version\":\"5.6.0\",\"minimum_index_compatibility_version\":\"5.0.0\"},\"tagline\":\"You Know, for Search\"}";
Map<String, String> paramMap = parseJsonStrToAbsolutePathKeyMap(json);
System.out.println(paramMap);
String str = "http:www.baidu.com?name=${name}&cn=${cluster_name}&vn=${version.number}";
System.out.println(StringUtils.replaceParam(str, paramMap));*/
String json = "{\"a\":{\"b\":{\"c\":\"d\"}},\"e\":{\"f\":\"1\",\"g\":\"2\"}}";
Map<String, Object> map = JSON.parseObject(json, new TypeReference<Map<String, Object>>() {
});
Set<String> keys = new HashSet<>();
parseMapLeafNodeKeys(keys, map, 1);
System.out.println(keys);
for (String key : keys) {
Object v = findValueWithLeafNodeKey(map, key, 1);
System.out.println(v);
}
}
}
suggest-task/src/main/java/com/secoo/so/suggest/util/PinYinUtils.java 0 → 100644
package com.secoo.so.suggest.util;
import com.github.stuxuhai.jpinyin.ChineseHelper;
import com.github.stuxuhai.jpinyin.PinyinFormat;
import com.github.stuxuhai.jpinyin.PinyinHelper;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
@Slf4j
public class PinYinUtils {
/**
* 简体转换为繁体
*/
public static String convertToTraditionalChinese(String str) {
String tempStr = null;
try {
tempStr = ChineseHelper.convertToTraditionalChinese(str);
} catch (Exception e) {
tempStr = str;
log.error("convertToTraditionalChinese error", e);
}
return tempStr;
}
/**
* 繁体转换为简体
*/
public static String convertToSimplifiedChinese(String str) {
String tempStr = null;
try {
tempStr = ChineseHelper.convertToSimplifiedChinese(str);
} catch (Exception e) {
tempStr = str;
log.error("convertToSimplifiedChinese error", e);
}
return tempStr;
}
/**
* 转换为有声调的拼音字符串
*
* @param str 汉字
* @return 有声调的拼音字符串
*/
public static String changeToToneMarkPinYin(String str) {
String tempStr = null;
try {
tempStr = PinyinHelper.convertToPinyinString(str, " ", PinyinFormat.WITH_TONE_MARK);
} catch (Exception e) {
log.error("changeToToneMarkPinYin error", e);
}
return tempStr;
}
/**
* 转换为数字声调字符串
*
* @param str 需转换的汉字
* @return 转换完成的拼音字符串
*/
public static String changeToToneNumberPinYin(String str) {
String tempStr = null;
try {
tempStr = PinyinHelper.convertToPinyinString(str, " ", PinyinFormat.WITH_TONE_NUMBER);
} catch (Exception e) {
log.error("changeToToneNumberPinYin error", e);
}
return tempStr;
}
/**
* 转换为不带音调的拼音字符串
*
* @param str 需转换的汉字
* @return 拼音字符串
*/
public static String changeToWithoutTonePinYin(String str) {
return changeToWithoutTonePinYin(str, " ");
}
/**
* 转换为不带音调的拼音字符串
*
* @param str 需转换的汉字
* @return 拼音字符串
*/
public static String changeToWithoutTonePinYinNoSeparator(String str) {
return changeToWithoutTonePinYin(str, "");
}
/**
* 转换为不带音调的拼音字符串
*
* @param str 需转换的汉字
* @return 拼音字符串
*/
public static String changeToWithoutTonePinYin(String str, String separator) {
String tempStr = null;
try {
tempStr = PinyinHelper.convertToPinyinString(str, separator, PinyinFormat.WITHOUT_TONE);
} catch (Exception e) {
log.error("changeToWithoutTonePinYin error: str:{}", str, e);
}
return tempStr;
}
/**
* 转换为每个汉字对应拼音首字母字符串
*
* @param str 需转换的汉字
* @return 拼音字符串
*/
public static String changeToGetShortPinYin(String str) {
String tempStr = null;
try {
tempStr = PinyinHelper.getShortPinyin(str);
} catch (Exception e) {
log.error("changeToGetShortPinYin error", e);
}
return tempStr;
}
/**
* 检查汉字是否为多音字
*
* @param str 需检查的汉字
* @return true 多音字，false 不是多音字
*/
public static boolean checkHasMultiPinyin(char str) {
boolean check = false;
try {
check = PinyinHelper.hasMultiPinyin(str);
} catch (Exception e) {
log.error("checkHasMultiPinyin error", e);
}
return check;
}
/**
* 是否是拼音
*
* @param str
* @return
*/
public static boolean isPinYin(String str) {
if (org.apache.commons.lang3.StringUtils.isBlank(str)) {
return false;
}
char[] chars = str.toCharArray();
for (char c : chars) {
boolean isPinYin = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
if (!isPinYin) {
return false;
}
}
return true;
}
/**
* 是否首字母是拼音
*/
public static boolean isFirstPinYin(String str) {
if (StringUtils.isBlank(str)) {
return false;
}
char[] chars = str.toCharArray();
char c = chars[0];
return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
}
/**
* 是否是英文词
*/
public static boolean isEnglishWord(String str) {
if (StringUtils.isBlank(str)) {
return false;
}
char[] chars = str.toCharArray();
for (char c : chars) {
boolean isValid = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || Character.isWhitespace(c);
if (!isValid) {
return false;
}
}
return true;
}
public static void main(String[] args) {
System.out.println(PinYinUtils.convertToTraditionalChinese("中国人"));
System.out.println(PinYinUtils.convertToSimplifiedChinese("中国人"));
System.out.println(PinYinUtils.changeToWithoutTonePinYin("中国人"));
System.out.println(PinYinUtils.changeToWithoutTonePinYinNoSeparator("博柏利 运动鞋"));
System.out.println(PinYinUtils.changeToWithoutTonePinYinNoSeparator("silk in"));
System.out.println(PinYinUtils.changeToWithoutTonePinYinNoSeparator("化妆品bb霜遮瑕"));
}
}
suggest-task/src/main/java/com/secoo/so/suggest/util/StringUtils.java 0 → 100644
package com.secoo.so.suggest.util;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import java.io.UnsupportedEncodingException;
import java.math.BigDecimal;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Created with IntelliJ IDEA. User: Administrator Date: 13-11-11 Time: 下午12:20
* To change this template use File | Settings | File Templates.
*/
public abstract class StringUtils {
private static final String FOLDER_SEPARATOR = "/";
private static final String WINDOWS_FOLDER_SEPARATOR = "\\";
private static final String TOP_PATH = "..";
private static final String CURRENT_PATH = ".";
private static final char EXTENSION_SEPARATOR = '.';
/**
* Represents a failed index search.
*
* @since 2.1
*/
public static final int INDEX_NOT_FOUND = -1;
// ---------------------------------------------------------------------
// General convenience methods for working with Strings
// ---------------------------------------------------------------------
/**
* Check that the given CharSequence is neither <code>null</code> nor of
* length 0. Note: Will return <code>true</code> for a CharSequence that
* purely consists of whitespace.
* <p>
*
* <pre>
* StringUtil.hasLength(null) = false
* StringUtil.hasLength("") = false
* StringUtil.hasLength(" ") = true
* StringUtil.hasLength("Hello") = true
* </pre>
*
* @param str the CharSequence to check (may be <code>null</code>)
* @return <code>true</code> if the CharSequence is not null and has length
* @see #hasText(String)
*/
public static boolean hasLength(CharSequence str) {
return (str != null && str.length() > 0);
}
/**
* Check that the given String is neither <code>null</code> nor of length 0.
* Note: Will return <code>true</code> for a String that purely consists of
* whitespace.
*
* @param str the String to check (may be <code>null</code>)
* @return <code>true</code> if the String is not null and has length
* @see #hasLength(CharSequence)
*/
public static boolean hasLength(String str) {
return hasLength((CharSequence) str);
}
/**
* Check whether the given CharSequence has actual text. More specifically,
* returns <code>true</code> if the string not <code>null</code>, its length
* is greater than 0, and it contains at least one non-whitespace character.
* <p>
*
* <pre>
* StringUtil.hasText(null) = false
* StringUtil.hasText("") = false
* StringUtil.hasText(" ") = false
* StringUtil.hasText("12345") = true
* StringUtil.hasText(" 12345 ") = true
* </pre>
*
* @param str the CharSequence to check (may be <code>null</code>)
* @return <code>true</code> if the CharSequence is not <code>null</code>,
* its length is greater than 0, and it does not contain whitespace
* only
* @see Character#isWhitespace
*/
public static boolean hasText(CharSequence str) {
if (!hasLength(str)) {
return false;
}
int strLen = str.length();
for (int i = 0; i < strLen; i++) {
if (!Character.isWhitespace(str.charAt(i))) {
return true;
}
}
return false;
}
/**
* Check whether the given String has actual text. More specifically,
* returns <code>true</code> if the string not <code>null</code>, its length
* is greater than 0, and it contains at least one non-whitespace character.
*
* @param str the String to check (may be <code>null</code>)
* @return <code>true</code> if the String is not <code>null</code>, its
* length is greater than 0, and it does not contain whitespace only
* @see #hasText(CharSequence)
*/
public static boolean hasText(String str) {
return hasText((CharSequence) str);
}
/**
* Check whether the given CharSequence contains any whitespace characters.
*
* @param str the CharSequence to check (may be <code>null</code>)
* @return <code>true</code> if the CharSequence is not empty and contains
* at least 1 whitespace character
* @see Character#isWhitespace
*/
public static boolean containsWhitespace(CharSequence str) {
if (!hasLength(str)) {
return false;
}
int strLen = str.length();
for (int i = 0; i < strLen; i++) {
if (Character.isWhitespace(str.charAt(i))) {
return true;
}
}
return false;
}
/**
* Check whether the given String contains any whitespace characters.
*
* @param str the String to check (may be <code>null</code>)
* @return <code>true</code> if the String is not empty and contains at
* least 1 whitespace character
* @see #containsWhitespace(CharSequence)
*/
public static boolean containsWhitespace(String str) {
return containsWhitespace((CharSequence) str);
}
/**
* Trim leading and trailing whitespace from the given String.
*
* @param str the String to check
* @return the trimmed String
* @see Character#isWhitespace
*/
public static String trimWhitespace(String str) {
if (!hasLength(str)) {
return str;
}
StringBuilder sb = new StringBuilder(str);
while (sb.length() > 0 && Character.isWhitespace(sb.charAt(0))) {
sb.deleteCharAt(0);
}
while (sb.length() > 0 && Character.isWhitespace(sb.charAt(sb.length() - 1))) {
sb.deleteCharAt(sb.length() - 1);
}
return sb.toString();
}
/**
* Trim <i>all</i> whitespace from the given String: leading, trailing, and
* inbetween characters.
*
* @param str the String to check
* @return the trimmed String
* @see Character#isWhitespace
*/
public static String trimAllWhitespace(String str) {
if (!hasLength(str)) {
return str;
}
StringBuilder sb = new StringBuilder(str);
int index = 0;
while (sb.length() > index) {
if (Character.isWhitespace(sb.charAt(index))) {
sb.deleteCharAt(index);
} else {
index++;
}
}
return sb.toString();
}
/**
* Trim leading whitespace from the given String.
*
* @param str the String to check
* @return the trimmed String
* @see Character#isWhitespace
*/
public static String trimLeadingWhitespace(String str) {
if (!hasLength(str)) {
return str;
}
StringBuilder sb = new StringBuilder(str);
while (sb.length() > 0 && Character.isWhitespace(sb.charAt(0))) {
sb.deleteCharAt(0);
}
return sb.toString();
}
/**
* Trim trailing whitespace from the given String.
*
* @param str the String to check
* @return the trimmed String
* @see Character#isWhitespace
*/
public static String trimTrailingWhitespace(String str) {
if (!hasLength(str)) {
return str;
}
StringBuilder sb = new StringBuilder(str);
while (sb.length() > 0 && Character.isWhitespace(sb.charAt(sb.length() - 1))) {
sb.deleteCharAt(sb.length() - 1);
}
return sb.toString();
}
/**
* Trim all occurences of the supplied leading character from the given
* String.
*
* @param str the String to check
* @param leadingCharacter the leading character to be trimmed
* @return the trimmed String
*/
public static String trimLeadingCharacter(String str, char leadingCharacter) {
if (!hasLength(str)) {
return str;
}
StringBuilder sb = new StringBuilder(str);
while (sb.length() > 0 && sb.charAt(0) == leadingCharacter) {
sb.deleteCharAt(0);
}
return sb.toString();
}
/**
* Trim all occurences of the supplied trailing character from the given
* String.
*
* @param str the String to check
* @param trailingCharacter the trailing character to be trimmed
* @return the trimmed String
*/
public static String trimTrailingCharacter(String str, char trailingCharacter) {
if (!hasLength(str)) {
return str;
}
StringBuilder sb = new StringBuilder(str);
while (sb.length() > 0 && sb.charAt(sb.length() - 1) == trailingCharacter) {
sb.deleteCharAt(sb.length() - 1);
}
return sb.toString();
}
/**
* Test if the given String starts with the specified prefix, ignoring
* upper/lower case.
*
* @param str the String to check
* @param prefix the prefix to look for
* @see String#startsWith
*/
public static boolean startsWithIgnoreCase(String str, String prefix) {
if (str == null || prefix == null) {
return false;
}
if (str.startsWith(prefix)) {
return true;
}
if (str.length() < prefix.length()) {
return false;
}
String lcStr = str.substring(0, prefix.length()).toLowerCase();
String lcPrefix = prefix.toLowerCase();
return lcStr.equals(lcPrefix);
}
/**
* Test if the given String ends with the specified suffix, ignoring
* upper/lower case.
*
* @param str the String to check
* @param suffix the suffix to look for
* @see String#endsWith
*/
public static boolean endsWithIgnoreCase(String str, String suffix) {
if (str == null || suffix == null) {
return false;
}
if (str.endsWith(suffix)) {
return true;
}
if (str.length() < suffix.length()) {
return false;
}
String lcStr = str.substring(str.length() - suffix.length()).toLowerCase();
String lcSuffix = suffix.toLowerCase();
return lcStr.equals(lcSuffix);
}
/**
* Test whether the given string matches the given substring at the given
* index.
*
* @param str the original string (or StringBuilder)
* @param index the index in the original string to start matching against
* @param substring the substring to match at the given index
*/
public static boolean substringMatch(CharSequence str, int index, CharSequence substring) {
for (int j = 0; j < substring.length(); j++) {
int i = index + j;
if (i >= str.length() || str.charAt(i) != substring.charAt(j)) {
return false;
}
}
return true;
}
/**
* Count the occurrences of the substring in string s.
*
* @param str string to search in. Return 0 if this is null.
* @param sub string to search for. Return 0 if this is null.
*/
public static int countOccurrencesOf(String str, String sub) {
if (str == null || sub == null || str.length() == 0 || sub.length() == 0) {
return 0;
}
int count = 0;
int pos = 0;
int idx;
while ((idx = str.indexOf(sub, pos)) != -1) {
++count;
pos = idx + sub.length();
}
return count;
}
/**
* Replace all occurences of a substring within a string with another
* string.
*
* @param inString String to examine
* @param oldPattern String to replace
* @param newPattern String to insert
* @return a String with the replacements
*/
public static String replace(String inString, String oldPattern, String newPattern) {
if (!hasLength(inString) || !hasLength(oldPattern) || newPattern == null) {
return inString;
}
StringBuilder sb = new StringBuilder();
int pos = 0; // our position in the old string
int index = inString.indexOf(oldPattern);
// the index of an occurrence we've found, or -1
int patLen = oldPattern.length();
while (index >= 0) {
sb.append(inString.substring(pos, index));
sb.append(newPattern);
pos = index + patLen;
index = inString.indexOf(oldPattern, pos);
}
sb.append(inString.substring(pos));
// remember to append any characters to the right of a match
return sb.toString();
}
/**
* Delete all occurrences of the given substring.
*
* @param inString the original String
* @param pattern the pattern to delete all occurrences of
* @return the resulting String
*/
public static String delete(String inString, String pattern) {
return replace(inString, pattern, "");
}
/**
* Delete any character in a given String.
*
* @param inString the original String
* @param charsToDelete a set of characters to delete. E.g. "az\n" will delete 'a's,
* 'z's and new lines.
* @return the resulting String
*/
public static String deleteAny(String inString, String charsToDelete) {
if (!hasLength(inString) || !hasLength(charsToDelete)) {
return inString;
}
StringBuilder sb = new StringBuilder();
for (int i = 0; i < inString.length(); i++) {
char c = inString.charAt(i);
if (charsToDelete.indexOf(c) == -1) {
sb.append(c);
}
}
return sb.toString();
}
// ---------------------------------------------------------------------
// Convenience methods for working with formatted Strings
// ---------------------------------------------------------------------
/**
* Quote the given String with single quotes.
*
* @param str the input String (e.g. "myString")
* @return the quoted String (e.g. "'myString'"), or
* <code>null<code> if the input was <code>null</code>
*/
public static String quote(String str) {
return (str != null ? "'" + str + "'" : null);
}
/**
* Turn the given Object into a String with single quotes if it is a String;
* keeping the Object as-is else.
*
* @param obj the input Object (e.g. "myString")
* @return the quoted String (e.g. "'myString'"), or the input object as-is
* if not a String
*/
public static Object quoteIfString(Object obj) {
return (obj instanceof String ? quote((String) obj) : obj);
}
/**
* Unqualify a string qualified by a '.' dot character. For example,
* "this.name.is.qualified", returns "qualified".
*
* @param qualifiedName the qualified name
*/
public static String unqualify(String qualifiedName) {
return unqualify(qualifiedName, '.');
}
/**
* Unqualify a string qualified by a separator character. For example,
* "this:name:is:qualified" returns "qualified" if using a ':' separator.
*
* @param qualifiedName the qualified name
* @param separator the separator
*/
public static String unqualify(String qualifiedName, char separator) {
return qualifiedName.substring(qualifiedName.lastIndexOf(separator) + 1);
}
/**
* Capitalize a <code>String</code>, changing the first letter to upper case
* as per {@link Character#toUpperCase(char)}. No other letters are changed.
*
* @param str the String to capitalize, may be <code>null</code>
* @return the capitalized String, <code>null</code> if null
*/
public static String capitalize(String str) {
return changeFirstCharacterCase(str, true);
}
/**
* Uncapitalize a <code>String</code>, changing the first letter to lower
* case as per {@link Character#toLowerCase(char)}. No other letters are
* changed.
*
* @param str the String to uncapitalize, may be <code>null</code>
* @return the uncapitalized String, <code>null</code> if null
*/
public static String uncapitalize(String str) {
return changeFirstCharacterCase(str, false);
}
private static String changeFirstCharacterCase(String str, boolean capitalize) {
if (str == null || str.length() == 0) {
return str;
}
StringBuilder sb = new StringBuilder(str.length());
if (capitalize) {
sb.append(Character.toUpperCase(str.charAt(0)));
} else {
sb.append(Character.toLowerCase(str.charAt(0)));
}
sb.append(str.substring(1));
return sb.toString();
}
/**
* Extract the filename from the given path, e.g. "mypath/myfile.txt" ->
* "myfile.txt".
*
* @param path the file path (may be <code>null</code>)
* @return the extracted filename, or <code>null</code> if none
*/
public static String getFilename(String path) {
if (path == null) {
return null;
}
int separatorIndex = path.lastIndexOf(FOLDER_SEPARATOR);
return (separatorIndex != -1 ? path.substring(separatorIndex + 1) : path);
}
/**
* Extract the filename extension from the given path, e.g.
* "mypath/myfile.txt" -> "txt".
*
* @param path the file path (may be <code>null</code>)
* @return the extracted filename extension, or <code>null</code> if none
*/
public static String getFilenameExtension(String path) {
if (path == null) {
return null;
}
int extIndex = path.lastIndexOf(EXTENSION_SEPARATOR);
if (extIndex == -1) {
return null;
}
int folderIndex = path.lastIndexOf(FOLDER_SEPARATOR);
if (folderIndex > extIndex) {
return null;
}
return path.substring(extIndex + 1);
}
/**
* Strip the filename extension from the given path, e.g.
* "mypath/myfile.txt" -> "mypath/myfile".
*
* @param path the file path (may be <code>null</code>)
* @return the path with stripped filename extension, or <code>null</code>
* if none
*/
public static String stripFilenameExtension(String path) {
if (path == null) {
return null;
}
int extIndex = path.lastIndexOf(EXTENSION_SEPARATOR);
if (extIndex == -1) {
return path;
}
int folderIndex = path.lastIndexOf(FOLDER_SEPARATOR);
if (folderIndex > extIndex) {
return path;
}
return path.substring(0, extIndex);
}
/**
* Apply the given relative path to the given path, assuming standard Java
* folder separation (i.e. "/" separators).
*
* @param path the path to start from (usually a full file path)
* @param relativePath the relative path to apply (relative to the full file path
* above)
* @return the full file path that results from applying the relative path
*/
public static String applyRelativePath(String path, String relativePath) {
int separatorIndex = path.lastIndexOf(FOLDER_SEPARATOR);
if (separatorIndex != -1) {
String newPath = path.substring(0, separatorIndex);
if (!relativePath.startsWith(FOLDER_SEPARATOR)) {
newPath += FOLDER_SEPARATOR;
}
return newPath + relativePath;
} else {
return relativePath;
}
}
/**
* Normalize the path by suppressing sequences like "path/.." and inner
* simple dots.
* <p>
* The result is convenient for path comparison. For other uses, notice that
* Windows separators ("\") are replaced by simple slashes.
*
* @param path the original path
* @return the normalized path
*/
public static String cleanPath(String path) {
if (path == null) {
return null;
}
String pathToUse = replace(path, WINDOWS_FOLDER_SEPARATOR, FOLDER_SEPARATOR);
// Strip prefix from path to analyze, to not treat it as part of the
// first path element. This is necessary to correctly parse paths like
// "file:core/../core/io/Resource.class", where the ".." should just
// strip the first "core" directory while keeping the "file:" prefix.
int prefixIndex = pathToUse.indexOf(":");
String prefix = "";
if (prefixIndex != -1) {
prefix = pathToUse.substring(0, prefixIndex + 1);
pathToUse = pathToUse.substring(prefixIndex + 1);
}
if (pathToUse.startsWith(FOLDER_SEPARATOR)) {
prefix = prefix + FOLDER_SEPARATOR;
pathToUse = pathToUse.substring(1);
}
String[] pathArray = delimitedListToStringArray(pathToUse, FOLDER_SEPARATOR);
List<String> pathElements = new LinkedList<String>();
int tops = 0;
for (int i = pathArray.length - 1; i >= 0; i--) {
String element = pathArray[i];
if (CURRENT_PATH.equals(element)) {
// Points to current directory - drop it.
} else if (TOP_PATH.equals(element)) {
// Registering top path found.
tops++;
} else {
if (tops > 0) {
// Merging path element with element corresponding to top
// path.
tops--;
} else {
// Normal path element found.
pathElements.add(0, element);
}
}
}
// Remaining top paths need to be retained.
for (int i = 0; i < tops; i++) {
pathElements.add(0, TOP_PATH);
}
return prefix + collectionToDelimitedString(pathElements, FOLDER_SEPARATOR);
}
/**
* Compare two paths after normalization of them.
*
* @param path1 first path for comparison
* @param path2 second path for comparison
* @return whether the two paths are equivalent after normalization
*/
public static boolean pathEquals(String path1, String path2) {
return cleanPath(path1).equals(cleanPath(path2));
}
/**
* Parse the given <code>localeString</code> value into a
* {@link Locale}.
* <p>
* This is the inverse operation of {@link Locale#toString
* Locale's toString}.
*
* @param localeString the locale string, following <code>Locale's</code>
* <code>toString()</code> format ("en", "en_UK", etc); also
* accepts spaces as separators, as an alternative to underscores
* @return a corresponding <code>Locale</code> instance
*/
public static Locale parseLocaleString(String localeString) {
String[] parts = tokenizeToStringArray(localeString, "_ ", false, false);
String language = (parts.length > 0 ? parts[0] : "");
String country = (parts.length > 1 ? parts[1] : "");
validateLocalePart(language);
validateLocalePart(country);
String variant = "";
if (parts.length >= 2) {
// There is definitely a variant, and it is everything after the
// country
// code sans the separator between the country code and the variant.
int endIndexOfCountryCode = localeString.indexOf(country) + country.length();
// Strip off any leading '_' and whitespace, what's left is the
// variant.
variant = trimLeadingWhitespace(localeString.substring(endIndexOfCountryCode));
if (variant.startsWith("_")) {
variant = trimLeadingCharacter(variant, '_');
}
}
return (language.length() > 0 ? new Locale(language, country, variant) : null);
}
private static void validateLocalePart(String localePart) {
for (int i = 0; i < localePart.length(); i++) {
char ch = localePart.charAt(i);
if (ch != '_' && ch != ' ' && !Character.isLetterOrDigit(ch)) {
throw new IllegalArgumentException("Locale part \"" + localePart + "\" contains invalid characters");
}
}
}
/**
* Determine the RFC 3066 compliant language tag, as used for the HTTP
* "Accept-Language" header.
*
* @param locale the Locale to transform to a language tag
* @return the RFC 3066 compliant language tag as String
*/
public static String toLanguageTag(Locale locale) {
return locale.getLanguage() + (hasText(locale.getCountry()) ? "-" + locale.getCountry() : "");
}
// ---------------------------------------------------------------------
// Convenience methods for working with String arrays
// ---------------------------------------------------------------------
/**
* Append the given String to the given String array, returning a new array
* consisting of the input array contents plus the given String.
*
* @param array the array to append to (can be <code>null</code>)
* @param str the String to append
* @return the new array (never <code>null</code>)
*/
public static String[] addStringToArray(String[] array, String str) {
if (ObjectUtils.isEmpty(array)) {
return new String[]{str};
}
String[] newArr = new String[array.length + 1];
System.arraycopy(array, 0, newArr, 0, array.length);
newArr[array.length] = str;
return newArr;
}
/**
* Concatenate the given String arrays into one, with overlapping array
* elements included twice.
* <p>
* The order of elements in the original arrays is preserved.
*
* @param array1 the first array (can be <code>null</code>)
* @param array2 the second array (can be <code>null</code>)
* @return the new array (<code>null</code> if both given arrays were
* <code>null</code>)
*/
public static String[] concatenateStringArrays(String[] array1, String[] array2) {
if (ObjectUtils.isEmpty(array1)) {
return array2;
}
if (ObjectUtils.isEmpty(array2)) {
return array1;
}
String[] newArr = new String[array1.length + array2.length];
System.arraycopy(array1, 0, newArr, 0, array1.length);
System.arraycopy(array2, 0, newArr, array1.length, array2.length);
return newArr;
}
/**
* Merge the given String arrays into one, with overlapping array elements
* only included once.
* <p>
* The order of elements in the original arrays is preserved (with the
* exception of overlapping elements, which are only included on their first
* occurrence).
*
* @param array1 the first array (can be <code>null</code>)
* @param array2 the second array (can be <code>null</code>)
* @return the new array (<code>null</code> if both given arrays were
* <code>null</code>)
*/
public static String[] mergeStringArrays(String[] array1, String[] array2) {
if (ObjectUtils.isEmpty(array1)) {
return array2;
}
if (ObjectUtils.isEmpty(array2)) {
return array1;
}
List<String> result = new ArrayList<String>();
result.addAll(Arrays.asList(array1));
for (String str : array2) {
if (!result.contains(str)) {
result.add(str);
}
}
return toStringArray(result);
}
/**
* Turn given source String array into sorted array.
*
* @param array the source array
* @return the sorted array (never <code>null</code>)
*/
public static String[] sortStringArray(String[] array) {
if (ObjectUtils.isEmpty(array)) {
return new String[0];
}
Arrays.sort(array);
return array;
}
/**
* Copy the given Collection into a String array. The Collection must
* contain String elements only.
*
* @param collection the Collection to copy
* @return the String array (<code>null</code> if the passed-in Collection
* was <code>null</code>)
*/
public static String[] toStringArray(Collection<String> collection) {
if (collection == null) {
return null;
}
return collection.toArray(new String[collection.size()]);
}
/**
* Copy the given Enumeration into a String array. The Enumeration must
* contain String elements only.
*
* @param enumeration the Enumeration to copy
* @return the String array (<code>null</code> if the passed-in Enumeration
* was <code>null</code>)
*/
public static String[] toStringArray(Enumeration<String> enumeration) {
if (enumeration == null) {
return null;
}
List<String> list = Collections.list(enumeration);
return list.toArray(new String[list.size()]);
}
/**
* Trim the elements of the given String array, calling
* <code>String.trim()</code> on each of them.
*
* @param array the original String array
* @return the resulting array (of the same size) with trimmed elements
*/
public static String[] trimArrayElements(String[] array) {
if (ObjectUtils.isEmpty(array)) {
return new String[0];
}
String[] result = new String[array.length];
for (int i = 0; i < array.length; i++) {
String element = array[i];
result[i] = (element != null ? element.trim() : null);
}
return result;
}
/**
* Remove duplicate Strings from the given array. Also sorts the array, as
* it uses a TreeSet.
*
* @param array the String array
* @return an array without duplicates, in natural sort order
*/
public static String[] removeDuplicateStrings(String[] array) {
if (ObjectUtils.isEmpty(array)) {
return array;
}
Set<String> set = new TreeSet<String>();
for (String element : array) {
set.add(element);
}
return toStringArray(set);
}
/**
* Split a String at the first occurrence of the delimiter. Does not include
* the delimiter in the result.
*
* @param toSplit the string to split
* @param delimiter to split the string up with
* @return a two element array with index 0 being before the delimiter, and
* index 1 being after the delimiter (neither element includes the
* delimiter); or <code>null</code> if the delimiter wasn't found in
* the given input String
*/
public static String[] split(String toSplit, String delimiter) {
if (!hasLength(toSplit) || !hasLength(delimiter)) {
return null;
}
int offset = toSplit.indexOf(delimiter);
if (offset < 0) {
return null;
}
String beforeDelimiter = toSplit.substring(0, offset);
String afterDelimiter = toSplit.substring(offset + delimiter.length());
return new String[]{beforeDelimiter, afterDelimiter};
}
public static List<String> splitToList(String toSplit, String delimiterRegex) {
List<String> strList = null;
if (!hasLength(toSplit) || !hasLength(delimiterRegex)) {
return strList;
}
String[] strs = toSplit.split(delimiterRegex);
strList = new ArrayList<String>();
for (String str : strs) {
strList.add(str);
}
return strList;
}
/**
* Take an array Strings and split each element based on the given
* delimiter. A <code>Properties</code> instance is then generated, with the
* left of the delimiter providing the key, and the right of the delimiter
* providing the value.
* <p>
* Will trim both the key and value before adding them to the
* <code>Properties</code> instance.
*
* @param array the array to process
* @param delimiter to split each element using (typically the equals symbol)
* @return a <code>Properties</code> instance representing the array
* contents, or <code>null</code> if the array to process was null
* or empty
*/
public static Properties splitArrayElementsIntoProperties(String[] array, String delimiter) {
return splitArrayElementsIntoProperties(array, delimiter, null);
}
/**
* Take an array Strings and split each element based on the given
* delimiter. A <code>Properties</code> instance is then generated, with the
* left of the delimiter providing the key, and the right of the delimiter
* providing the value.
* <p>
* Will trim both the key and value before adding them to the
* <code>Properties</code> instance.
*
* @param array the array to process
* @param delimiter to split each element using (typically the equals symbol)
* @param charsToDelete one or more characters to remove from each element prior to
* attempting the split operation (typically the quotation mark
* symbol), or <code>null</code> if no removal should occur
* @return a <code>Properties</code> instance representing the array
* contents, or <code>null</code> if the array to process was
* <code>null</code> or empty
*/
public static Properties splitArrayElementsIntoProperties(String[] array, String delimiter, String charsToDelete) {
if (ObjectUtils.isEmpty(array)) {
return null;
}
Properties result = new Properties();
for (String element : array) {
if (charsToDelete != null) {
element = deleteAny(element, charsToDelete);
}
String[] splittedElement = split(element, delimiter);
if (splittedElement == null) {
continue;
}
result.setProperty(splittedElement[0].trim(), splittedElement[1].trim());
}
return result;
}
/**
* Tokenize the given String into a String array via a StringTokenizer.
* Trims tokens and omits empty tokens.
* <p>
* The given delimiters string is supposed to consist of any number of
* delimiter characters. Each of those characters can be used to separate
* tokens. A delimiter is always a single character; for multi-character
* delimiters, consider using <code>delimitedListToStringArray</code>
*
* @param str the String to tokenize
* @param delimiters the delimiter characters, assembled as String (each of those
* characters is individually considered as delimiter).
* @return an array of the tokens
* @see StringTokenizer
* @see String#trim()
* @see #delimitedListToStringArray
*/
public static String[] tokenizeToStringArray(String str, String delimiters) {
return tokenizeToStringArray(str, delimiters, true, true);
}
/**
* Tokenize the given String into a String array via a StringTokenizer.
* <p>
* The given delimiters string is supposed to consist of any number of
* delimiter characters. Each of those characters can be used to separate
* tokens. A delimiter is always a single character; for multi-character
* delimiters, consider using <code>delimitedListToStringArray</code>
*
* @param str the String to tokenize
* @param delimiters the delimiter characters, assembled as String (each of those
* characters is individually considered as delimiter)
* @param trimTokens trim the tokens via String's <code>trim</code>
* @param ignoreEmptyTokens omit empty tokens from the result array (only applies to
* tokens that are empty after trimming; StringTokenizer will not
* consider subsequent delimiters as token in the first place).
* @return an array of the tokens (<code>null</code> if the input String was
* <code>null</code>)
* @see StringTokenizer
* @see String#trim()
* @see #delimitedListToStringArray
*/
public static String[] tokenizeToStringArray(String str, String delimiters, boolean trimTokens,
boolean ignoreEmptyTokens) {
if (str == null) {
return null;
}
StringTokenizer st = new StringTokenizer(str, delimiters);
List<String> tokens = new ArrayList<String>();
while (st.hasMoreTokens()) {
String token = st.nextToken();
if (trimTokens) {
token = token.trim();
}
if (!ignoreEmptyTokens || token.length() > 0) {
tokens.add(token);
}
}
return toStringArray(tokens);
}
/**
* Take a String which is a delimited list and convert it to a String array.
* <p>
* A single delimiter can consists of more than one character: It will still
* be considered as single delimiter string, rather than as bunch of
* potential delimiter characters - in contrast to
* <code>tokenizeToStringArray</code>.
*
* @param str the input String
* @param delimiter the delimiter between elements (this is a single delimiter,
* rather than a bunch individual delimiter characters)
* @return an array of the tokens in the list
* @see #tokenizeToStringArray
*/
public static String[] delimitedListToStringArray(String str, String delimiter) {
return delimitedListToStringArray(str, delimiter, null);
}
/**
* Take a String which is a delimited list and convert it to a String array.
* <p>
* A single delimiter can consists of more than one character: It will still
* be considered as single delimiter string, rather than as bunch of
* potential delimiter characters - in contrast to
* <code>tokenizeToStringArray</code>.
*
* @param str the input String
* @param delimiter the delimiter between elements (this is a single delimiter,
* rather than a bunch individual delimiter characters)
* @param charsToDelete a set of characters to delete. Useful for deleting unwanted
* line breaks: e.g. "\r\n\f" will delete all new lines and line
* feeds in a String.
* @return an array of the tokens in the list
* @see #tokenizeToStringArray
*/
public static String[] delimitedListToStringArray(String str, String delimiter, String charsToDelete) {
if (str == null) {
return new String[0];
}
if (delimiter == null) {
return new String[]{str};
}
List<String> result = new ArrayList<String>();
if ("".equals(delimiter)) {
for (int i = 0; i < str.length(); i++) {
result.add(deleteAny(str.substring(i, i + 1), charsToDelete));
}
} else {
int pos = 0;
int delPos;
while ((delPos = str.indexOf(delimiter, pos)) != -1) {
result.add(deleteAny(str.substring(pos, delPos), charsToDelete));
pos = delPos + delimiter.length();
}
if (str.length() > 0 && pos <= str.length()) {
// Add rest of String, but not in case of empty input.
result.add(deleteAny(str.substring(pos), charsToDelete));
}
}
return toStringArray(result);
}
/**
* Convert a CSV list into an array of Strings.
*
* @param str the input String
* @return an array of Strings, or the empty array in case of empty input
*/
public static String[] commaDelimitedListToStringArray(String str) {
return delimitedListToStringArray(str, ",");
}
/**
* Convenience method to convert a CSV string list to a set. Note that this
* will suppress duplicates.
*
* @param str the input String
* @return a Set of String entries in the list
*/
public static Set<String> commaDelimitedListToSet(String str) {
Set<String> set = new TreeSet<String>();
String[] tokens = commaDelimitedListToStringArray(str);
for (String token : tokens) {
set.add(token);
}
return set;
}
/**
* Convenience method to return a Collection as a delimited (e.g. CSV)
* String. E.g. useful for <code>toString()</code> implementations.
*
* @param coll the Collection to display
* @param delim the delimiter to use (probably a ",")
* @param prefix the String to start each element with
* @param suffix the String to end each element with
* @return the delimited String
*/
public static String collectionToDelimitedString(Collection<?> coll, String delim, String prefix, String suffix) {
if (CollectionUtils.isEmpty(coll)) {
return "";
}
StringBuilder sb = new StringBuilder();
Iterator<?> it = coll.iterator();
while (it.hasNext()) {
sb.append(prefix).append(it.next()).append(suffix);
if (it.hasNext()) {
sb.append(delim);
}
}
return sb.toString();
}
/**
* Convenience method to return a Collection as a delimited (e.g. CSV)
* String. E.g. useful for <code>toString()</code> implementations.
*
* @param coll the Collection to display
* @param delim the delimiter to use (probably a ",")
* @return the delimited String
*/
public static String collectionToDelimitedString(Collection<?> coll, String delim) {
return collectionToDelimitedString(coll, delim, "", "");
}
/**
* Convenience method to return a Collection as a CSV String. E.g. useful
* for <code>toString()</code> implementations.
*
* @param coll the Collection to display
* @return the delimited String
*/
public static String collectionToCommaDelimitedString(Collection<?> coll) {
return collectionToDelimitedString(coll, ",");
}
/**
* Convenience method to return a String array as a delimited (e.g. CSV)
* String. E.g. useful for <code>toString()</code> implementations.
*
* @param arr the array to display
* @param delim the delimiter to use (probably a ",")
* @return the delimited String
*/
public static String arrayToDelimitedString(Object[] arr, String delim) {
if (ObjectUtils.isEmpty(arr)) {
return "";
}
if (arr.length == 1) {
return ObjectUtils.nullSafeToString(arr[0]);
}
StringBuilder sb = new StringBuilder();
for (int i = 0; i < arr.length; i++) {
if (i > 0) {
sb.append(delim);
}
sb.append(arr[i]);
}
return sb.toString();
}
/**
* Convenience method to return a String array as a CSV String. E.g. useful
* for <code>toString()</code> implementations.
*
* @param arr the array to display
* @return the delimited String
*/
public static String arrayToCommaDelimitedString(Object[] arr) {
return arrayToDelimitedString(arr, ",");
}
public static String[] getMiddle(String uri) {
String[] temp = uri.split(FOLDER_SEPARATOR);
return temp;
}
// Empty checks
//-----------------------------------------------------------------------
/**
* <p>Checks if a CharSequence is empty ("") or null.</p>
*
* <pre>
* StringUtils.isEmpty(null) = true
* StringUtils.isEmpty("") = true
* StringUtils.isEmpty(" ") = false
* StringUtils.isEmpty("bob") = false
* StringUtils.isEmpty(" bob ") = false
* </pre>
*
* <p>NOTE: This method changed in Lang version 2.0.
* It no longer trims the CharSequence.
* That functionality is available in isBlank().</p>
*
* @param cs the CharSequence to check, may be null
* @return {@code true} if the CharSequence is empty or null
* @since 3.0 Changed signature from isEmpty(String) to isEmpty(CharSequence)
*/
public static boolean isEmpty(final CharSequence cs) {
return cs == null || cs.length() == 0;
}
/**
* @param s
* @return 如果<tt>s</tt>为<tt>null</tt>或空白字符串返回<tt>true</tt>
*/
public static boolean isBlank(String s) {
return s == null || s.trim().length() == 0;
}
public static boolean isNotBlank(String... strs) {
boolean isNotNull = true;
if (strs.length == 0) {
return false;
}
for (String str : strs) {
if (StringUtils.isBlank(str)) {
isNotNull = false;
break;
}
}
return isNotNull;
}
/**
* 截取字符串，按照系统默认的字符集，截取后的后缀为“...”
*
* @param target 被截取的原字符串，此方法执行前会先<tt>trim</tt>
* @param maxBytes 截取后字符串的最大<tt>byte</tt>数，包括截取后的字符串的后缀
* @return
* @see #substring(String, String, int, String)
*/
public static String substring(String target, int maxBytes) {
return substring(target.trim(), Charset.defaultCharset().name(), maxBytes, "...");
}
/**
* 截取字符串
*
* @param target 被截取的原字符串
* @param charset 字符串的字符集
* @param maxBytes 截取后字符串的最大<tt>byte</tt>数，包括截取后的字符串的后缀
* @param append 字符串被截去后的后缀
* @return
*/
public static String substring(String target, String charset, int maxBytes, String append) {
try {
int count = getBytes(target, charset).length;
if (count <= maxBytes) {
return target;
} else {
int bytesCount = 0;
char[] replace = new char[getBytes(append, charset).length];
int j = 0;
int bound = maxBytes - getBytes(append, charset).length;
for (int i = 0; i < target.length(); i++) {
char c = target.charAt(i);
bytesCount = c > 255 ? bytesCount + 2 : bytesCount + 1;
if (bytesCount > maxBytes) {
return target.substring(0, i - j).concat(append);
}
if (bytesCount > bound) {
replace[j++] = c;
}
}
}
} catch (UnsupportedEncodingException e) {
throw new RuntimeException(e);
}
throw new RuntimeException("Unreachable!");
}
public static String substring(String src, int start_idx, int end_idx) {
byte[] b = src.getBytes();
String tgt = "";
for (int i = start_idx; i <= end_idx; i++) {
tgt += (char) b[i];
}
return tgt;
}
private static byte[] getBytes(String s, String charset) throws UnsupportedEncodingException {
return s.getBytes(charset);
}
public static String GBKToUTF(String str) {
String utfStr = null;
try {
utfStr = new String(str.getBytes("GBK"), StandardCharsets.UTF_8);
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
}
return utfStr;
}
public static String UTFToGBK(String str) {
String utfStr = null;
try {
utfStr = new String(str.getBytes(StandardCharsets.UTF_8), "GBK");
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
}
return utfStr;
}
public static String ISOToGBK(String str) {
String utfStr = null;
try {
utfStr = new String(str.getBytes("ISO8859_1"), "GBK");
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
}
return utfStr;
}
// public static String getPasWordStr(String password) {
// String passStr = "";
// MessageDigest digester = null;
// try {
// digester = MessageDigest.getInstance("SHA");
// digester.update(password.getBytes("utf-8"));
// byte[] bytes = digester.digest();
// passStr = new String(Hex.encodeHex(bytes));
// } catch (Exception e) {
// e.printStackTrace();
// }
// return passStr;
// }
/**
* 将字符转换成map {row:20000,pageno:1,startdate:2008-9-10,enddate:2008-10-1}
*
* @return
* @author hudaowan
* @date Nov 3, 2008 10:03:26 AM
*/
public static Map<String, String> strToMap(String str) {
Map<String, String> map = new HashMap<String, String>();
if (!isBlank(str)) {
String strObj = str.substring(1, str.length() - 1);
String[] obj = strObj.split(",");
for (String value : obj) {
String[] key = value.split(":");
map.put(key[0], key[1]);
}
}
return map;
}
/**
* 将map转换成字符
*
* @param map
* @return
* @author hudaowan
* @date Nov 3, 2008 10:21:31 AM
*/
public static String mapToStr(Map<String, String> map) {
String str = "";
if (map != null) {
for (Map.Entry<String, String> obj : map.entrySet()) {
String key = obj.getKey();
String value = obj.getValue();
str += key + ":" + value + ",";
}
}
if (str.length() > 0) {
str = "{" + str.substring(0, str.length() - 1) + "}";
}
return str;
}
/**
* 用<tt>seperator</tt>连接字符串,例如将数组{"a","b","c"}使用';'连接,得到"a;b;c",忽略
* <tt>null<tt>和空白字符串
* <p>
* &#64;see #join(String[], String, boolean, boolean)
* &#64;param s 需要连接的字符串数组
* &#64;param seperator 分隔符
* &#64;return 连接好的字符串或""
*
* @throws NullPointerException 如果<tt>s</tt>或<tt>seperator</tt>为<tt>null</tt>
*/
public static String join(String[] s, String seperator) {
return join(s, seperator, true, true);
}
/**
* 用<tt>seperator</tt>连接字符串,例如将字符串列表使用'，'连接,得到"a,b,c",忽略 <tt>null<tt>和空白字符串
* <p>
* &#64;see #join(List<String>, String, boolean, boolean)
* &#64;param strList 需要连接的字符串列表
* &#64;param seperator 分隔符
* &#64;return 连接好的字符串或""
*
* @throws NullPointerException 如果<tt>strList</tt>或<tt>seperator</tt>为<tt>null</tt>
*/
public static String join(List<String> strList, String seperator) {
return join(strList, seperator, true, true);
}
/**
* 用<tt>seperator</tt>连接字符串,例如将数组{"a","b","c"}使用';'连接，得到"a;b;c"
*
* @param s 需要连接的字符串数组
* @param seperator 分隔符
* @param ignoreBlank 是否忽略空字符串,通过<tt>String.trim().length() == 0</tt>判断空字符串
* @param ignoreNull 是否忽略<tt>null</tt>
* @return 连接好的字符串或""
* @throws NullPointerException 如果<tt>s</tt>或<tt>seperator</tt>为<tt>null</tt>
*/
public static String join(String[] s, String seperator, boolean ignoreBlank, boolean ignoreNull) {
if (s == null || seperator == null)
throw new NullPointerException();
StringBuilder result = new StringBuilder(256);
for (String s_ : s) {
if (ignoreNull && s_ == null)
continue;
else if (ignoreBlank && s_.trim().length() == 0)
continue;
result.append(s_);
result.append(seperator);
}
int i = result.length();
if (i > 0)
return result.substring(0, i - seperator.length());
else
return "";
}
/**
* 用<tt>seperator</tt>连接字符串,例如将字符串列表使用'，'连接,得到"a,b,c",忽略
*
* @param strList 需要连接的字符串数组
* @param seperator 分隔符
* @param ignoreBlank 是否忽略空字符串,通过<tt>String.trim().length() == 0</tt>判断空字符串
* @param ignoreNull 是否忽略<tt>null</tt>
* @return 连接好的字符串或""
* @throws NullPointerException 如果<tt>s</tt>或<tt>seperator</tt>为<tt>null</tt>
*/
public static String join(List<String> strList, String seperator, boolean ignoreBlank, boolean ignoreNull) {
if (strList == null || seperator == null)
throw new NullPointerException();
StringBuilder result = new StringBuilder(256);
for (String s_ : strList) {
if (ignoreNull && s_ == null)
continue;
else if (ignoreBlank && s_.trim().length() == 0)
continue;
result.append(s_);
result.append(seperator);
}
int i = result.length();
if (i > 0)
return result.substring(0, i - seperator.length());
else
return "";
}
public static <T> String join(Collection<T> list, String seperator) {
if (list == null || seperator == null)
throw new NullPointerException();
StringBuilder result = new StringBuilder(256);
for (T t : list) {
if (t == null)
continue;
else if (t instanceof String && ((String) t).trim().length() == 0)
continue;
result.append(t);
result.append(seperator);
}
int i = result.length();
if (i > 0)
return result.substring(0, i - seperator.length());
else
return "";
}
/**
* 给列表中每个元素的前后增加一样的指定字符
*
* @param list
* @param speChar
* @return
*/
public static List<String> concatSpeChar(Collection<String> list, String speChar) {
return concatSpeChar(list, speChar, speChar);
}
/**
* 给列表中每个元素的前后增加指定字符
*
* @param list
* @param beforChar
* @param endChar
* @return
*/
public static List<String> concatSpeChar(Collection<String> list, String beforChar, String endChar) {
if (null == list) {
return Collections.EMPTY_LIST;
}
List<String> result = new ArrayList<>();
for (String s : list) {
result.add(beforChar + s + endChar);
}
return result;
}
/**
* 将CamelCase（驼峰）转换成大写字母，以“_”为间隔，例如abcFoo转换成ABC_FOO
*
* @param s
* @return
*/
public static String camelToUnderlineUpper(String s) {
final String pattern = "[A-Z]*[a-z0-9]+|[A-Z0-9]+";
Pattern p = Pattern.compile(pattern); // the expression
Matcher m = p.matcher(s); // the source
String r = null;
while (m.find()) {
if (r != null) {
r = r + "_" + m.group().toUpperCase();
} else {
r = m.group().toUpperCase();
}
}
return r;
}
/**
* 将CamelCase（驼峰）转换成小写字母，以“_”为间隔，例如abcFoo转换成abc_foo
*
* @param s
* @return
*/
public static String camelToUnderlineLower(String s) {
String r = camelToUnderlineUpper(s);
return r != null ? r.toLowerCase() : null;
}
/**
* 将下划线分隔字母转换成CamelCase，以“_”为间隔，例如ABC_FOO转换成abcFoo
*
* @param s
* @return
*/
public static String underlineToCamel(String s) {
String[] tokens = s.split("_");
String r = tokens[0].toLowerCase();
for (int i = 1; i < tokens.length; i++) {
r += (tokens[i].substring(0, 1).toUpperCase() + tokens[i].substring(1).toLowerCase());
}
return r;
}
public static String trim(String str) {
str = str.replace('　', ' ');
return str.trim();
}
public static Object convertStringToDataType(String value, String dataType) {
// 如果结果返回null，向上也返回null，不要返回0，因为0也是一个值不同于null
if ("INTEGER".equals(dataType)) {
return Integer.valueOf(value);
} else if ("LONG".equals(dataType)) {
return Long.valueOf(value);
} else if ("DOUBLE".equals(dataType) || "FLOAT".equals(dataType) || "PERCENT".equals(dataType)
|| "CURRENCY".equals(dataType) || "TIME".equals(dataType)) {
BigDecimal fixValue = new BigDecimal(value);
return fixValue.setScale(2, BigDecimal.ROUND_HALF_UP).doubleValue(); // 四舍五入，保留两位小数
} else {
return value;
}
}
/**
* 根据数字生成excel列表头，最大到zz
*
* @param i
* @return
*/
public static String i2s(int i) {
String s = "A B C D E F G H I J K L M N O P Q R S T U V W X Y Z";
String[] sArray = s.split(" ");
if (i < 1)
return "";
if ((i / 26) == 0)
return sArray[i % 26 - 1];
else {
if (i % 26 == 0)
return (i2s((i / 26) - 1)) + sArray[26 - 1];
else
return sArray[(i / 26) - 1] + sArray[i % 26 - 1];
}
}
/**
* 根据数字生成excel列表头，最大受int最大值限制
*
* @param num
* @return
*/
public static String getExcelColumnLabel(int num) {
String temp = "";
double i = Math.floor(Math.log(25.0 * (num) / 26.0 + 1) / Math.log(26)) + 1;
if (i > 1) {
double sub = num - 26 * (Math.pow(26, i - 1) - 1) / 25;
for (double j = i; j > 0; j--) {
temp = temp + (char) (sub / Math.pow(26, j - 1) + 65);
sub = sub % Math.pow(26, j - 1);
}
} else {
temp = temp + (char) (num + 65);
}
return temp;
}
public static String firstChartoUpperCase(String str) {
return str.replaceFirst(str.substring(0, 1), str.substring(0, 1).toUpperCase());
}
/**
* 科学技术法转为正常数字，例如：1.24E7转为12400000
*
* @param str
* @return
*/
public static String kxjsConvert(String str) {
if (isKxjs(str)) {
BigDecimal db = new BigDecimal(str);
return db.toPlainString();
} else {
return str;
}
}
/**
* 验证当前字符串是否是科学计数法
*
* @param str
* @return
*/
public static boolean isKxjs(String str) {
return str != null && str.matches("^((-?\\d+\\.?\\d+)[Ee]{1}([-+]?\\d+))$");
}
public static boolean isNumber(String str) {
return StringUtils.isNotBlank(str) && str.matches("^(\\-|\\+)?\\d+(\\.\\d+)?$");
}
public static boolean isBigDecimal(Object value) {
try {
if (value != null) {
new BigDecimal(String.valueOf(value));
return true;
}
} catch (Exception e) {
}
return false;
}
public static boolean isDateStr(String str, String dateFormat) {
boolean convertSuccess = true;
SimpleDateFormat format = new SimpleDateFormat(dateFormat, Locale.US);
try {
format.setLenient(false);// 设置lenient为false.
// 否则SimpleDateFormat会比较宽松地验证日期，比如2007/02/29会被接受，并转换成2007/03/01
format.parse(str);
} catch (Exception e) {
convertSuccess = false;
}
return convertSuccess;
}
/**
* 转义字符串中的正则表达式使用的特殊字符
*
* @param str
* @return
* @date: 2016年7月23日
* @author peng.xu
*/
public static String escapeRegexp(String str) {
if (StringUtils.isNotBlank(str)) {
String[] charArray = new String[]{"\\", "*", ".", "?", "+", "$", "^", "[", "]", "(", ")", "{", "}", "|",
"!"};
for (String s : charArray) {
str = str.replace(s, "\\" + s);
}
}
return str;
}
/**
* 获取json字符串中对应属性的值
*
* @param jsonStr
* @param key
* @param jsonRegex json对应的正则表达式规则， 如果不为空则需要校验正则匹配
* @return
* @date: 2016年8月15日
* @author peng.xu
*/
public static String getValueOfJson(String jsonStr, String key, String jsonRegex) {
String value = null;
if (StringUtils.isNotBlank(jsonStr) && StringUtils.isNotBlank(key)) {
if (StringUtils.isBlank(jsonRegex) || (StringUtils.isNotBlank(jsonRegex) && jsonStr.matches(jsonRegex))) {
try {
Object valueObj = JSON.parseObject(jsonStr).get(key);
if (valueObj != null) {
value = valueObj.toString();
}
} catch (Exception e) {
e.printStackTrace();
}
}
}
return value;
}
/**
* 比较字符串大小，优先按照字母序，再按照大小写排序
*/
public static int compareStringIgnoreCase(String str1, String str2) {
int result = 0;
if (StringUtils.isBlank(str1)) {
result = StringUtils.isBlank(str2) ? 0 : -1;
} else if (StringUtils.isBlank(str2)) {
result = 1;
} else {
result = str1.toUpperCase().compareToIgnoreCase(str2);
if (result == 0) {
result = str1.compareTo(str2);
}
}
return result;
}
/**
* 根据指定长度 分隔字符串
*
* @param str 需要处理的字符串
* @param length 分隔长度
* @return 字符串集合
*/
public static List<String> splitString(String str, int length) {
List<String> list = new ArrayList<String>();
for (int i = 0; i < str.length(); i += length) {
int endIndex = i + length;
if (endIndex <= str.length()) {
list.add(str.substring(i, i + length));
} else {
list.add(str.substring(i, str.length() - 1));
}
}
return list;
}
/**
* 使用map中值替换字符串中的${key}参数
*
* @date: 2018-12-07
* @author xupeng
*/
public static String replaceParam(String str, Map<String, ?> paramMap) {
if (StringUtils.isNotBlank(str) && CollectionUtils.isNotEmpty(paramMap)) {
for (Map.Entry<String, ?> entry : paramMap.entrySet()) {
String key = entry.getKey();
String value = entry.getValue() != null ? String.valueOf(entry.getValue()) : null;
if (StringUtils.isNotBlank(value)) {
value = Matcher.quoteReplacement(value);
str = str.replaceAll("\\$\\{" + key + "\\}", value);
}
}
}
return str;
}
/**
* 替换掉所有控制字符,范围[\x00-\x1F\x7F]即ASCII码字符0-31加127共33个字符
*
* @author xupeng
* @date: 2019-01-25
*/
public static String replaceControlChar(String str) {
return str != null ? str.replaceAll("\\p{Cntrl}", "") : null;
}
/**
* 按照正则提取字符串
*
* @param regex
* @param source
* @param index
* @return
*/
public static String getMatcher(String regex, String source, int index) {
String result = "";
Pattern pattern = Pattern.compile(regex);
Matcher matcher = pattern.matcher(source);
while (matcher.find()) {
result = matcher.group(index);
}
return result;
}
public static String replaceMysql(String content) {
if (isBlank(content)) {
return content;
}
content = content.replace("\\", "\\\\");
content = content.replace("'", "\\'");
content = content.replace("\"", "\\\"");
return content;
}
/**
* 首字母大写
*
* @param var
* @return
*/
public static String captureName(String var) {
if (isBlank(var)) {
return var;
}
char[] cs = var.toCharArray();
char first = cs[0];
//只有在 a-z的范围内才做转换
if (first >= 97 && first <= 122) {
cs[0] -= 32;
return String.valueOf(cs);
}
return var;
}
public static List<String> getImgs(String content) {
String img = "";
Pattern p_image;
Matcher m_image;
List<String> images = new ArrayList<>();
String regEx_img = "(<img.*src\\s*=\\s*(.*?)[^>]*?>)";
p_image = Pattern.compile(regEx_img, Pattern.CASE_INSENSITIVE);
m_image = p_image.matcher(content);
while (m_image.find()) {
img = m_image.group();
Matcher m = Pattern.compile("src\\s*=\\s*\"?(.*?)(\"|>|\\s+)").matcher(img);
while (m.find()) {
images.add(m.group(1));
}
}
return images;
}
public static JSONObject safeParseJSONObject(String json) {
try {
return JSON.parseObject(json);
} catch (Exception e) {
e.printStackTrace();
}
return null;
}
public static <T> T safeParseObject(String json, Class<T> clazz) {
try {
return JSON.parseObject(json, clazz);
} catch (Exception e) {
e.printStackTrace();
}
return null;
}
public static <T> List<T> safeParseObjectList(String json, Class<T> clazz) {
try {
return JSON.parseArray(json, clazz);
} catch (Exception e) {
e.printStackTrace();
}
return null;
}
private static final Map<String, String> ChCharToEnMapping = new HashMap() {{
put("，", ",");
put("：", ":");
put("。", ".");
put("！", "!");
put("？", "?");
put("【", "[");
put("】", "]");
put("（", "(");
put("）", ")");
put("‘", "'");
put("’", "'");
put("“", "\"");
put("”", "\"");
}};
/**
* 全角转半角
*/
public static String dbc2Sbc(String str) {
StringBuilder buf = new StringBuilder();
if (StringUtils.isNotBlank(str)) {
for (int i = 0; i < str.length(); i++) {
char ch = str.charAt(i);
String chStr = String.valueOf(ch);
if (ChCharToEnMapping.containsKey(chStr)) { // 中文字符转英文字符
buf.append(ChCharToEnMapping.get(chStr));
} else if (ch >= 32 && ch <= 126) {
buf.append(chStr);
} else if (ch == 12288) { // 全角空格
ch = 32;
buf.append(ch); // 半角空格
} else {
ch -= 65248; // 全角半角转换间隔
String fixChStr = String.valueOf(ch);
if (ChCharToEnMapping.containsKey(fixChStr)) { // 中文字符转英文字符
buf.append(ChCharToEnMapping.get(fixChStr));
} else if (ch >= 32 && ch <= 126) {
buf.append(fixChStr);
} else {
buf.append(chStr);
}
}
}
}
return buf.toString();
}
public static int str2Int(String str) {
if (StringUtils.isNumber(str)) {
return Integer.parseInt(str);
}
return 0;
}
private static Set<Character> special = new HashSet<Character>() {{
add(' ');
add('，');
add(',');
add('!');
add('@');
add('#');
add('$');
add('%');
add('^');
add('&');
add('*');
add('(');
add(')');
add('！');
add('@');
add('#');
add('￥');
add('%');
add('*');
add('（');
add('）');
add('‘');
add('[');
add(']');
add('{');
add('}');
add('?');
add('？');
add('，');
add('。');
add('、');
add('《');
add('》');
add('；');
add('<');
add('>');
add('|');
add('.');
add('-');
add('_');
add('—');
add('·');
add('\ufffc'); // 在实现textView的富文本时，如果添加一张图片后，如果直接发送textView的内容时，图片会被字符串“\U0000fffc”替换
}};
/**
* 判断一个字符是否是特殊字符（special 里面的字符）
*/
private static boolean isSpecialCharacter(Character c) {
if (special.contains(c)) {
return false;
}
return true;
}
/**
* 把一个字符里面的特殊字符剔除掉
*/
public static String removeSpecialChar(String word) {
if (word == null) {
return word;
}
StringBuffer sb = new StringBuffer();
char[] chars = word.toCharArray();
for (int i = 0; i < chars.length; i++) {
if (isSpecialCharacter(chars[i])) {
sb.append(chars[i]);
}
}
return sb.toString();
}
/**
* 逗号等分隔符转空格，去两边空格，中间多个空格转一个空格
*/
public static String cleanMultiBlank(String str) {
// 去两边空格，转小写，中间多个空格转一个空格
if (str != null) {
String clean = str
.trim()
.replaceAll("\\s{1,}", " ");
return clean;
}
return null;
}
/**
* 判断是否是英文字符串
*/
public static boolean isEnStr(String word) {
boolean result = word.matches("[a-zA-Z]+");
return result;
}
/**
* 判断是否包含中文
*/
public static boolean isContainChStr(String word) {
Pattern p = Pattern.compile("[\u4e00-\u9fa5]");
Matcher m = p.matcher(word);
if (m.find()) {
return true;
}
return false;
}
/**
* 计算中英文字符串的字节长度 <br/>
* 一个中文占3个字节
*
* @param str
* @return int 字符串的字节长度
*/
public static int getByteLength(String str) {
return getByteLength(str, "UTF-8");
}
/**
* 计算中英文字符串的字节长度 <br/>
* 一个中文占3个字节
*
* @param str
* @return int 字符串的字节长度
*/
public static int getByteLength(String str, String charset) {
if (str == null || str.length() == 0) {
return 0;
}
try {
return str.getBytes(charset).length;
} catch (UnsupportedEncodingException e) {
System.out.println("计算中英文字符串的字节长度失败，");
e.printStackTrace();
}
return 0;
}
/**
* 32位md5加密
*/
public static String md5(String str) {
if (str == null) {
return null;
}
String result = "";
try {
MessageDigest md5 = MessageDigest.getInstance("MD5");
md5.update(str.getBytes("UTF-8"));
byte b[] = md5.digest();
StringBuffer buf = new StringBuffer("");
int i = 0;
for (int offset = 0; offset < b.length; offset++) {
i = b[offset];
if (i < 0) {
i += 256;
}
if (i < 16) {
buf.append("0");
}
buf.append(Integer.toHexString(i));
}
result = buf.toString();
} catch (NoSuchAlgorithmException | UnsupportedEncodingException e) {
System.out.println("encode md5 error");
e.printStackTrace();
}
return result;
}
/**
* <p>Strips any of a set of characters from the start and end of a String.
* This is similar to {@link String#trim()} but allows the characters
* to be stripped to be controlled.</p>
*
* <p>A {@code null} input String returns {@code null}.
* An empty string ("") input returns the empty string.</p>
*
* <p>If the stripChars String is {@code null}, whitespace is
* stripped as defined by {@link Character#isWhitespace(char)}.
* Alternatively use {@link #strip(String)}.</p>
*
* <pre>
* StringUtils.strip(null, *) = null
* StringUtils.strip("", *) = ""
* StringUtils.strip("abc", null) = "abc"
* StringUtils.strip(" abc", null) = "abc"
* StringUtils.strip("abc ", null) = "abc"
* StringUtils.strip(" abc ", null) = "abc"
* StringUtils.strip(" abcyx", "xyz") = " abc"
* </pre>
*
* @param str the String to remove characters from, may be null
* @param stripChars the characters to remove, null treated as whitespace
* @return the stripped String, {@code null} if null String input
*/
public static String strip(String str, final String stripChars) {
if (isEmpty(str)) {
return str;
}
str = stripStart(str, stripChars);
return stripEnd(str, stripChars);
}
/**
* <p>Strips any of a set of characters from the start of a String.</p>
*
* <p>A {@code null} input String returns {@code null}.
* An empty string ("") input returns the empty string.</p>
*
* <p>If the stripChars String is {@code null}, whitespace is
* stripped as defined by {@link Character#isWhitespace(char)}.</p>
*
* <pre>
* StringUtils.stripStart(null, *) = null
* StringUtils.stripStart("", *) = ""
* StringUtils.stripStart("abc", "") = "abc"
* StringUtils.stripStart("abc", null) = "abc"
* StringUtils.stripStart(" abc", null) = "abc"
* StringUtils.stripStart("abc ", null) = "abc "
* StringUtils.stripStart(" abc ", null) = "abc "
* StringUtils.stripStart("yxabc ", "xyz") = "abc "
* </pre>
*
* @param str the String to remove characters from, may be null
* @param stripChars the characters to remove, null treated as whitespace
* @return the stripped String, {@code null} if null String input
*/
public static String stripStart(final String str, final String stripChars) {
int strLen;
if (str == null || (strLen = str.length()) == 0) {
return str;
}
int start = 0;
if (stripChars == null) {
while (start != strLen && Character.isWhitespace(str.charAt(start))) {
start++;
}
} else if (stripChars.isEmpty()) {
return str;
} else {
while (start != strLen && stripChars.indexOf(str.charAt(start)) != INDEX_NOT_FOUND) {
start++;
}
}
return str.substring(start);
}
/**
* <p>Strips any of a set of characters from the end of a String.</p>
*
* <p>A {@code null} input String returns {@code null}.
* An empty string ("") input returns the empty string.</p>
*
* <p>If the stripChars String is {@code null}, whitespace is
* stripped as defined by {@link Character#isWhitespace(char)}.</p>
*
* <pre>
* StringUtils.stripEnd(null, *) = null
* StringUtils.stripEnd("", *) = ""
* StringUtils.stripEnd("abc", "") = "abc"
* StringUtils.stripEnd("abc", null) = "abc"
* StringUtils.stripEnd(" abc", null) = " abc"
* StringUtils.stripEnd("abc ", null) = "abc"
* StringUtils.stripEnd(" abc ", null) = " abc"
* StringUtils.stripEnd(" abcyx", "xyz") = " abc"
* StringUtils.stripEnd("120.00", ".0") = "12"
* </pre>
*
* @param str the String to remove characters from, may be null
* @param stripChars the set of characters to remove, null treated as whitespace
* @return the stripped String, {@code null} if null String input
*/
public static String stripEnd(final String str, final String stripChars) {
int end;
if (str == null || (end = str.length()) == 0) {
return str;
}
if (stripChars == null) {
while (end != 0 && Character.isWhitespace(str.charAt(end - 1))) {
end--;
}
} else if (stripChars.isEmpty()) {
return str;
} else {
while (end != 0 && stripChars.indexOf(str.charAt(end - 1)) != INDEX_NOT_FOUND) {
end--;
}
}
return str.substring(0, end);
}
}
suggest-task/src/main/profiles/prod/config.properties 0 → 100644
# suggestTask
suggestTask.prefixFilterList=["https://", "http://", "dg", "d & g", "dolce&gabbana","dolce & gabbana", "杜嘉班纳", "避孕", "情趣", "cucci", "乒乓球", "cuccl", "gucii","tod's","iwc7"]
suggestTask.ManualFolder=/data/pssmaster/corpus_set/suggest_corpus/manual
suggestTask.SensitiveFolder=/data/pssmaster/corpus_set/suggest_corpus/sensitive
suggestTask.EuropeWordFolder=/data/pssmaster/corpus_set/suggest_corpus/europe_word
suggestTask.batchSize=10000
suggestTask.threadPoolSize=10
suggestTask.searchWordWarningCount=1000000
suggestTask.suggestTagMaxSize=5
suggestTask.warningPhones=13426233960
suggestTask.es.url=http://bigdataescluster.secoolocal.com:9200
suggestTask.es.user=search
suggestTask.es.password=search5z0NvEn1D
suggestTask.es.index=search_suggest_index
suggestTask.es.type=search_suggest_type
suggestTask.es.batchSize=2000
suggest-task/src/main/profiles/prod/db.properties 0 → 100644
erp.read.url=jdbc:mysql://192.168.50.40:3306/secooErpDB?useUnicode=true&amp;characterEncoding=utf8&amp;noAccessToProcedureBodies=true&amp;zeroDateTimeBehavior=convertToNull&amp;allowMultiQueries=true
erp.read.user=so_Erp_R
erp.read.password=5RgzudyyFlApTmve
seo.read.url=jdbc:mysql://secooSeoDB.master.com:3307/secooSeoDB?useUnicode=true&amp;characterEncoding=utf8&amp;zeroDateTimeBehavior=convertToNull
seo.read.user=sem_Seo_W
seo.read.password=C2IiHfNKYpT1onsR
dw.read.url=jdbc:mysql://secooDataWarehouse.slave.com:3306/secooDataWarehouse?useUnicode=true&amp;characterEncoding=utf8&amp;zeroDateTimeBehavior=convertToNull
dw.read.user=Search_DataWar_R
dw.read.password=pY1P9zUj9x1M65ot5szo
\ No newline at end of file
suggest-task/src/main/profiles/test/config.properties 0 → 100644
# suggestTask
suggestTask.prefixFilterList=["https://", "http://", "dg", "d & g", "dolce&gabbana","dolce & gabbana", "杜嘉班纳", "避孕", "情趣", "cucci", "乒乓球", "cuccl", "gucii","tod's","iwc7"]
suggestTask.ManualFolder=/data/pssmaster/corpus_set/suggest_corpus/manual
suggestTask.SensitiveFolder=/data/pssmaster/corpus_set/suggest_corpus/sensitive
suggestTask.EuropeWordFolder=/data/pssmaster/corpus_set/suggest_corpus/europe_word
suggestTask.batchSize=10000
suggestTask.threadPoolSize=10
suggestTask.suggestTagMaxSize=5
suggestTask.searchWordWarningCount=1000000
suggestTask.es.url=http://10.0.254.139:9200
suggestTask.es.user=suggest
suggestTask.es.password=suggest456
suggestTask.es.index=search_suggest_index
suggestTask.es.type=search_suggest_type
suggestTask.es.batchSize=2000
\ No newline at end of file
suggest-task/src/main/profiles/test/db.properties 0 → 100644
erp.read.url=jdbc:mysql://10.4.3.223:3306/secooErpDB?useUnicode=true&amp;characterEncoding=utf8&amp;noAccessToProcedureBodies=true&amp;zeroDateTimeBehavior=convertToNull&amp;allowMultiQueries=true
erp.read.user=3306_test
erp.read.password=iS6CXpYqgZ8Mhjui
seo.read.url=jdbc:mysql://10.4.3.223:3306/secooSeoDB?useUnicode=true&amp;characterEncoding=utf8&amp;zeroDateTimeBehavior=convertToNull
seo.read.user=SeoDB_test
seo.read.password=Cxkfq57huej0fTpK
\ No newline at end of file
suggest-task/src/main/resources/log4j.properties 0 → 100644
log4j.rootLogger=INFO, FILE
# logger
log4j.logger.com.secoo.so.mysql.binlog=INFO
log4j.logger.org.apache.zookeeper=WARN
log4j.logger.com.alibaba.dubbo=WARN
log4j.appender.FILE=org.apache.log4j.RollingFileAppender
log4j.appender.FILE.File=logs/service.log
log4j.appender.FILE.MaxFileSize=100MB
log4j.appender.FILE.MaxBackupIndex=5
log4j.appender.FILE.Threshold=INFO
log4j.appender.FILE.layout=org.apache.log4j.PatternLayout
log4j.appender.FILE.layout.ConversionPattern=[%d{yyyy-MM-dd HH:mm:ss}]%.8t %-5p %c:%L %m%n
suggest-task/src/main/resources/search_suggest_index.txt 0 → 100644
{
"state": "open",
"settings": {
"index": {
"number_of_shards": "1",
"provided_name": "search_suggest_index",
"creation_date": "1551702662623",
"analysis": {
"analyzer": {
"suggest_analyzer": {
"tokenizer": "suggest_tokenizer"
}
},
"tokenizer": {
"suggest_tokenizer": {
"type": "edge_ngram",
"min_gram": "1",
"max_gram": "20"
}
}
},
"number_of_replicas": "2",
"uuid": "GdxvBgzsSICrpSddf6bqIQ",
"version": {
"created": "6040099"
}
}
},
"mappings": {
"search_suggest_type": {
"properties": {
"isEuropeWord": {
"type": "boolean"
},
"yearCount": {
"type": "integer"
},
"yearCartRatio": {
"type": "double"
},
"weekClickRatio": {
"type": "double"
},
"weekCount": {
"type": "integer"
},
"wordABRank": {
"type": "float"
},
"IsEuropeWord": {
"type": "boolean"
},
"analyzer": {
"type": "text",
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
}
},
"weekClickCount": {
"type": "integer"
},
"text": {
"type": "text",
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
}
},
"keyword": {
"analyzer": "suggest_analyzer",
"type": "text",
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
}
},
"isManual": {
"type": "boolean"
},
"keywordPinYin": {
"analyzer": "suggest_analyzer",
"type": "text",
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
}
},
"query": {
"properties": {
"bool": {
"properties": {
"must": {
"properties": {
"term": {
"properties": {
"keywordVersion": {
"type": "date"
}
}
}
}
}
}
}
}
},
"weekCartRatio": {
"type": "double"
},
"yearClickCount": {
"type": "integer"
},
"updateTime": {
"type": "long"
},
"yearCartCount": {
"type": "integer"
},
"keywordVersion": {
"type": "keyword"
},
"yearClickRatio": {
"type": "double"
},
"isCategory": {
"type": "boolean"
},
"field": {
"type": "text",
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
}
},
"isSensitive": {
"type": "boolean"
},
"suggestTags": {
"type": "keyword"
},
"weekCartCount": {
"type": "integer"
},
"doc": {
"properties": {
"isManual": {
"type": "boolean"
},
"keywordPinYin": {
"type": "text",
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
}
},
"weekCartRatio": {
"type": "float"
},
"yearClickCount": {
"type": "long"
},
"updateTime": {
"type": "long"
},
"yearCartCount": {
"type": "long"
},
"yearCount": {
"type": "long"
},
"yearCartRatio": {
"type": "float"
},
"weekClickRatio": {
"type": "float"
},
"weekCount": {
"type": "long"
},
"wordABRank": {
"type": "float"
},
"keywordVersion": {
"type": "date"
},
"yearClickRatio": {
"type": "float"
},
"isCategory": {
"type": "boolean"
},
"isSensitive": {
"type": "boolean"
},
"suggestTags": {
"type": "text",
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
}
},
"weekClickCount": {
"type": "long"
},
"weekCartCount": {
"type": "long"
},
"wordRank": {
"type": "float"
},
"keyword": {
"type": "text",
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
}
},
"manualValue": {
"type": "long"
},
"isBrand": {
"type": "boolean"
}
}
},
"wordRank": {
"type": "double"
},
"manualValue": {
"type": "integer"
},
"isBrand": {
"type": "boolean"
}
}
}
},
"aliases": [
],
"primary_terms": {
"0": 3
},
"in_sync_allocations": {
"0": [
"rmaxShfDRkCpdv91Iz4nkQ",
"tWXAarrcTQmXvB07MuWLYg",
"lcW9Sv9MTgSkKb9XmmUzOQ"
]
}
}
\ No newline at end of file
suggest-task/src/test/java/com/secoo/so/suggest/TestCode.java 0 → 100644
package com.secoo.so.suggest;
import com.alibaba.fastjson.JSON;
import com.secoo.so.suggest.entity.EsSuggestKeywordInfo;
import com.secoo.so.suggest.es.EsClient;
import com.secoo.so.suggest.es.EsObject;
import com.secoo.so.suggest.task.SuggestTask;
import com.secoo.so.suggest.util.FileUtils;
import com.secoo.so.suggest.util.PinYinUtils;
import com.secoo.so.suggest.util.StringUtils;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import java.io.File;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.List;
/**
* @author xupeng
* @date: 2022/2/8
*/
public class TestCode {
public static void main7(String[] args) throws Exception {
System.out.println(StringUtils.dbc2Sbc("模糊字母"));
System.out.println(PinYinUtils.convertToSimplifiedChinese("模糊字母"));
}
public static void main(String[] args) throws Exception {
List<String> esLines = FileUtils.readLines(new File("d:\\suggest-es.json"));
List<String> newLines = FileUtils.readLines(new File("d:\\suggest-index-keyword-20220209142219.txt"));
int count = 0;
for (String esLine : esLines) {
if (!newLines.contains(esLine)) {
System.out.println(++count + "\tonlyEs: " + esLine);
}
}
count = 0;
for (String newLine : newLines) {
if (!esLines.contains(newLine)) {
System.out.println(++count + "\tonlyNew: " + newLine);
}
}
}
public static void main5(String[] args) throws Exception {
EsClient esClient = EsClient.buildEsClient("http://yunhead.siku.cn", "search", "search5z0NvEn1D");
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
List<EsSuggestKeywordInfo> list = esClient.search("search_suggest_index", queryBuilder, EsSuggestKeywordInfo.class);
StringBuilder keywordBuilder = new StringBuilder();
for (EsSuggestKeywordInfo suggestKeywordInfo : list) {
keywordBuilder.append(suggestKeywordInfo.getKeyword()).append("\n");
}
FileUtils.saveToFile(keywordBuilder.toString(), "d:\\suggest-es.json", false);
esClient.close();
}
public static void main4(String[] args) throws Exception {
EsClient esClient = EsClient.buildEsClient("http://yunhead.siku.cn", "search", "search5z0NvEn1D");
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery()
.must(QueryBuilders.termQuery("keyword", "高跟鞋女秋冬新款"));
List<EsSuggestKeywordInfo> list = esClient.search("search_suggest_index", queryBuilder, EsSuggestKeywordInfo.class);
System.out.println(JSON.toJSONString(list));
List<EsObject> esList = new ArrayList<>();
for (EsSuggestKeywordInfo esSuggestKeywordInfo : list) {
esSuggestKeywordInfo.setKeyword(esSuggestKeywordInfo.getKeyword() + "红色");
esList.add(new EsObject(StringUtils.md5(esSuggestKeywordInfo.getKeyword()), esSuggestKeywordInfo));
}
esClient.batch("search_suggest_index", "search_suggest_type", esList);
list = esClient.search("search_suggest_index", queryBuilder, EsSuggestKeywordInfo.class);
System.out.println(JSON.toJSONString(list));
esClient.close();
}
public static void main3(String[] args) {
String json = "{\"isBrand\":false,\"isCategory\":false,\"isEuropeWord\":false,\"isManual\":false,\"isSensitive\":false,\"keyword\":\"高跟鞋女秋冬新款\",\"keywordPinYin\":\"gaogenxienvqiudongxinkuan\",\"keywordVersion\":\"2022-02-07\",\"manualValue\":0,\"suggestTags\":\"\",\"updateTime\":1644287409631,\"weekCartCount\":0,\"weekCartRatio\":0.0,\"weekClickCount\":0,\"weekClickRatio\":0.0,\"weekCount\":0,\"wordABRank\":21251.870636624943,\"wordRank\":19772.059911048906,\"yearCartCount\":1,\"yearCartRatio\":0.1111111111111111,\"yearClickCount\":33,\"yearClickRatio\":1.2222222222222223,\"yearCount\":27}";
EsSuggestKeywordInfo suggestKeywordInfo = JSON.parseObject(json, EsSuggestKeywordInfo.class);
System.out.println(SuggestTask.calculateWordRank(suggestKeywordInfo));
suggestKeywordInfo.setKeyword(suggestKeywordInfo.getKeyword() + "红色");
}
public static void main2(String[] args) {
String word = StringUtils.dbc2Sbc("我爱。.·").replaceAll("\ufffc|,|，|\\.", " ");
System.out.println(word);
}
public static void main1(String[] args) {
// TODO Auto-generated method stub
try {
System.out.println("中文a".getBytes("UTF-8").length); // 7
System.out.println("中文a".getBytes("GBK").length); // 7
System.out.println(StringUtils.getByteLength("中文a", "utf-8")); // 7
System.out.println(StringUtils.getByteLength("中文a", "gBK")); // 7
System.out.println(StringUtils.getByteLength("中文a")); // 7
System.out.println("中文a".length()); // 3
} catch (UnsupportedEncodingException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment