Commit 8bdd2e20 by xupeng

init project

parent 60d8f5e5
......@@ -2,4 +2,47 @@
main/info.log
main/test.go
main/suggest-task
main/suggest-task.exe
\ No newline at end of file
main/suggest-task.exe
# Compiled class file
**/target
**/*.class
**/.classpath
**/.settings
**/.project
**/.idea
**/.vscode
**/*.iml
**/.DS_Store
**/node_modules
**/dist
*.suo
*.ntvs*
*.njsproj
*.sln
*.sublime-project
*.sublime-workspace
**/coverage
**/test/unit/coverage/
**/test/e2e/reports/
# Log file
*.log
**/*.log
# BlueJ files
*.ctxt
# Mobile Tools for Java (J2ME)
.mtj.tmp/
# Package Files #
*.jar
*.war
*.ear
*.zip
*.tar.gz
*.rar
# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
**/hs_err_pid*
\ No newline at end of file
<assembly>
<id>dependency</id>
<formats>
<format>jar</format>
</formats>
<!-- 压缩包下是否生成和项目名相同的根目录-->
<includeBaseDirectory>false</includeBaseDirectory>
<dependencySets>
<dependencySet>
<!-- 是否把当前项目的输出jar包并使用,true则会把当前项目输出为jar包到输出目录,false不输出 -->
<useProjectArtifact>false</useProjectArtifact>
<!-- 是否解压依赖包 -->
<unpack>true</unpack>
<scope>system</scope>
</dependencySet>
<dependencySet>
<!-- 是否解压依赖包 -->
<unpack>true</unpack>
<!-- 将scope为runtime的依赖包打包 -->
<scope>runtime</scope>
<excludes>
</excludes>
<includes>
</includes>
</dependencySet>
</dependencySets>
<fileSets>
<fileSet>
<directory>${project.build.outputDirectory}</directory>
<outputDirectory>/</outputDirectory>
</fileSet>
</fileSets>
</assembly>
\ No newline at end of file
#!/usr/bin/env bash
java -cp so-suggest-task-1.0-SNAPSHOT.jar com.secoo.so.suggest.task.SuggestTask
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.secoo.so</groupId>
<artifactId>so-suggest-task</artifactId>
<version>1.0-SNAPSHOT</version>
<name>so-suggest-task</name>
<profiles>
<profile>
<id>prod</id>
<build>
<resources>
<resource>
<directory>src/main/profiles/prod</directory>
</resource>
</resources>
</build>
</profile>
<profile>
<id>test</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<build>
<resources>
<resource>
<directory>src/main/profiles/test</directory>
</resource>
</resources>
</build>
</profile>
</profiles>
<dependencies>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.36</version>
</dependency>
<dependency>
<groupId>commons-dbcp</groupId>
<artifactId>commons-dbcp</artifactId>
<version>1.4</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.16.6</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.12</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.12</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.5</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.alibaba/fastjson -->
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.58</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpasyncclient</artifactId>
<version>4.1.1</version>
</dependency>
<dependency>
<groupId>com.secoo.search.third-patry</groupId>
<artifactId>third-patry-jpinyin</artifactId>
<version>1.1.8</version>
</dependency>
</dependencies>
<build>
<resources>
<resource>
<directory>src/main/resources</directory>
</resource>
</resources>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.6.0</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
<encoding>UTF-8</encoding>
</configuration>
</plugin>
<!-- 打成 withDependencies jar 包-->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.6</version>
<configuration>
<!-- not append assembly id in release file name -->
<appendAssemblyId>false</appendAssemblyId>
<descriptors>
<descriptor>assembly.xml</descriptor>
</descriptors>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
package com.secoo.so.suggest.config;
import com.alibaba.fastjson.JSON;
import com.secoo.so.suggest.util.StringUtils;
import lombok.extern.slf4j.Slf4j;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
@Slf4j
public class ConfigUtil {
private static final String CONFIG_FILE = "config.properties";
private static ConfigUtil INST;
private Properties prop;
private ConfigUtil() {
prop = load();
}
public synchronized static void init() {
ConfigUtil configUtil = new ConfigUtil();
INST = configUtil;
}
private Properties load() {
Properties tmp = new Properties();
try {
log.info(this.getClass().getClassLoader().getResource(CONFIG_FILE).getPath());
tmp.load(new InputStreamReader(this.getClass().getClassLoader().getResourceAsStream(CONFIG_FILE), "UTF-8"));
} catch (IOException e) {
log.error(e.getMessage(), e);
}
return tmp;
}
public static void printAll() {
if (INST == null) {
init();
}
log.info("[" + CONFIG_FILE + "] =============== start print config properties ===============");
if (INST != null && INST.prop != null) {
for (Map.Entry<Object, Object> entry : INST.prop.entrySet()) {
log.info("[" + CONFIG_FILE + "] " + entry.getKey() + "=" + entry.getValue());
}
}
log.info("[" + CONFIG_FILE + "] =============== end print config properties ===============");
}
public static String getString(String key) {
if (INST == null) {
init();
}
return INST.prop.getProperty(key);
}
public static String getString(String key, String defaultValue) {
String val = getString(key);
if (val == null) {
return defaultValue;
}
return val;
}
public static int getInt(String key, int defaultValue) {
String val = getString(key);
if (val != null) {
try {
return Integer.parseInt(val);
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
return defaultValue;
}
private static List<String> prefixFilterList = null;
public static List<String> getPrefixFilterList() {
if (prefixFilterList == null) {
String val = getString("prefix_filter_list");
if (StringUtils.isNotBlank(val)) {
prefixFilterList = JSON.parseArray(val, String.class);
}
prefixFilterList = new ArrayList<>();
}
return prefixFilterList;
}
}
package com.secoo.so.suggest.db;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.dbcp.BasicDataSource;
import java.io.IOException;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.Properties;
@Slf4j
public class DwDataSource {
private DwDataSource() {
}
private static class DataSourceHolder {
private static BasicDataSource dataSource = new BasicDataSource();
static {
Properties prop = new Properties();
try {
prop.load(DwDataSource.class.getClassLoader().getResourceAsStream("db.properties"));
} catch (IOException e) {
log.error("init config error", e);
}
dataSource.setDriverClassName("com.mysql.jdbc.Driver");
dataSource.setUrl(prop.getProperty("dw.read.url"));
dataSource.setUsername(prop.getProperty("dw.read.user"));
dataSource.setPassword(prop.getProperty("dw.read.password"));
dataSource.setTestWhileIdle(true);
dataSource.setTestOnReturn(true);
dataSource.setTimeBetweenEvictionRunsMillis(30000);
dataSource.setMaxActive(30);
dataSource.setTestOnBorrow(true);
dataSource.setValidationQuery("select 1 from dual");
}
}
public static Connection getConnection() {
try {
return DataSourceHolder.dataSource.getConnection();
} catch (SQLException e) {
log.error("获取链接异常", e);
throw new RuntimeException("获取链接异常:" + e.getMessage(), e);
}
}
public static void close() {
try {
DataSourceHolder.dataSource.close();
} catch (SQLException e) {
log.error("close error", e);
}
}
}
package com.secoo.so.suggest.db;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.dbcp.BasicDataSource;
import java.io.IOException;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.Properties;
@Slf4j
public class ErpDataSource {
private ErpDataSource() {
}
private static class DataSourceHolder {
private static BasicDataSource dataSource = new BasicDataSource();
static {
Properties prop = new Properties();
try {
prop.load(ErpDataSource.class.getClassLoader().getResourceAsStream("db.properties"));
} catch (IOException e) {
log.error("init config error", e);
}
dataSource.setDriverClassName("com.mysql.jdbc.Driver");
dataSource.setUrl(prop.getProperty("erp.read.url"));
dataSource.setUsername(prop.getProperty("erp.read.user"));
dataSource.setPassword(prop.getProperty("erp.read.password"));
dataSource.setTestWhileIdle(true);
dataSource.setTestOnReturn(true);
dataSource.setTimeBetweenEvictionRunsMillis(30000);
dataSource.setMaxActive(30);
dataSource.setTestOnBorrow(true);
dataSource.setValidationQuery("select 1 from dual");
}
}
public static Connection getConnection() {
try {
return DataSourceHolder.dataSource.getConnection();
} catch (SQLException e) {
log.error("获取链接异常", e);
throw new RuntimeException("获取链接异常:" + e.getMessage(), e);
}
}
public static void close() {
try {
DataSourceHolder.dataSource.close();
} catch (SQLException e) {
log.error("close error", e);
}
}
}
package com.secoo.so.suggest.db;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.dbcp.BasicDataSource;
import java.io.IOException;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.Properties;
@Slf4j
public class SeoDataSource {
private SeoDataSource() {
}
private static class DataSourceHolder {
private static BasicDataSource dataSource = new BasicDataSource();
static {
Properties prop = new Properties();
try {
prop.load(ErpDataSource.class.getClassLoader().getResourceAsStream("db.properties"));
} catch (IOException e) {
log.error("init config error", e);
}
dataSource.setDriverClassName("com.mysql.jdbc.Driver");
dataSource.setUrl(prop.getProperty("seo.read.url"));
dataSource.setUsername(prop.getProperty("seo.read.user"));
dataSource.setPassword(prop.getProperty("seo.read.password"));
dataSource.setTestWhileIdle(true);
dataSource.setTestOnReturn(true);
dataSource.setTimeBetweenEvictionRunsMillis(30000);
dataSource.setMaxActive(30);
dataSource.setTestOnBorrow(true);
dataSource.setValidationQuery("select 1 from dual");
}
}
public static Connection getConnection() {
try {
return DataSourceHolder.dataSource.getConnection();
} catch (SQLException e) {
log.error("获取链接异常", e);
throw new RuntimeException("获取链接异常:" + e.getMessage(), e);
}
}
public static void close() {
try {
DataSourceHolder.dataSource.close();
} catch (SQLException e) {
log.error("close error", e);
}
}
}
package com.secoo.so.suggest.db;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class SuggestTaskDao {
private static final Logger log = LoggerFactory.getLogger(SuggestTaskDao.class);
/**
* 查询
*/
// public static List<SearchRelateHotWordInfo> querySearchRelateInfo(SearchRelateHotWordInfo relateHotWordInfo){
// List<SearchRelateHotWordInfo> searchRelateHotWordInfoList = Lists.newArrayList();
// if (relateHotWordInfo == null){
// return searchRelateHotWordInfoList;
// }
// String brandCategoryId = relateHotWordInfo.getBrandCategoryId();
// if (StringUtils.isBlank(brandCategoryId)){
// return searchRelateHotWordInfoList;
// }
// Connection conn = SEODataSource.getConnection();
// PreparedStatement stmt = null;
// ResultSet rs = null;
// String sql = "SELECT category_code,category_code_name,brand_id,brand_name,brand_category_id FROM secooSeoDB.t_search_relate_hot_word WHERE brand_category_id =? and is_del = 0";
// try {
// stmt = conn.prepareStatement(sql);
// stmt.setString(1, brandCategoryId);
// rs = stmt.executeQuery();
// while (rs.next()){
// SearchRelateHotWordInfo searchRelateHotWordInfo = new SearchRelateHotWordInfo();
// String brandCategoryIdStr = rs.getString("brand_category_id");
// if (StringUtils.isBlank(brandCategoryIdStr)){
// continue;
// }
// searchRelateHotWordInfo.setBrandCategoryId(brandCategoryIdStr);
// searchRelateHotWordInfoList.add(searchRelateHotWordInfo);
// }
// }catch (Exception e){
// log.error("querySearchRelateInfo select exception", e);
// }finally {
// DBConnection.close(conn, stmt, rs);
// }
// return searchRelateHotWordInfoList;
// }
}
package com.secoo.so.suggest.helper;
import com.secoo.so.suggest.db.ErpDataSource;
import com.secoo.so.suggest.util.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.Statement;
import java.util.HashSet;
import java.util.Set;
/**
* 加载敏感词汇
*/
public class AdvWordsHelper {
private static final Logger LOG = LoggerFactory.getLogger(AdvWordsHelper.class);
/**
* 敏感词key
*/
public static final String ADV_WORDS = "so.advWords";
/**
* 汉字匹配
*/
public static final String regEx1 = "[\\u4e00-\\u9fa5]";
private static final String SENSITIVE_SQL = "SELECT sensitive_words FROM t_sensitive_info WHERE status=1 and del_flag=0";
/**
* 获取敏感词
*/
public static String getAdvWords() {
Connection conn = ErpDataSource.getConnection();
Statement stmt = null;
ResultSet rs = null;
Set<String> advWordSet = new HashSet<>();
try {
stmt = conn.createStatement();
rs = stmt.executeQuery(SENSITIVE_SQL);
while (rs.next()) {
String sensitiveWord = rs.getString(1);
if (StringUtils.isNotEmpty(sensitiveWord.trim())) {
advWordSet.add(sensitiveWord);
}
}
} catch (Exception e) {
LOG.error("获取敏感词异常: " + e.getMessage(), e);
} finally {
ObjectUtils.safeClose(conn, stmt, rs);
}
if (advWordSet.size() > 0) {
return StringUtils.join(advWordSet, ",");
}
return null;
}
}
package com.secoo.so.suggest.task;
import com.secoo.so.suggest.config.ConfigUtil;
import lombok.extern.slf4j.Slf4j;
/**
* 初始化suggest搜索词到es索引
*/
@Slf4j
public class SuggestTask {
public static void main(String[] args) {
long start = System.currentTimeMillis();
log.info(">>>>>>>>>>>> start run SuggestTask");
// 初始化配置信息
ConfigUtil.init();
ConfigUtil.printAll();
log.info("<<<<<<<<<<<< end run SuggestTask, cost: {}ms", (System.currentTimeMillis() - start));
}
}
package com.secoo.so.suggest.util;
import com.alibaba.fastjson.JSON;
import org.apache.http.HttpStatus;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.net.URI;
import java.util.*;
import java.util.concurrent.*;
public class FeiShuUtil {
private static final Logger LOGGER = LoggerFactory.getLogger(FeiShuUtil.class);
private static CloseableHttpClient client = HttpClientBuilder.create().build();
private static final URI FEI_SHU_URL = URI.create("http://matrix-inform.secoolocal.com/user/sendToUser");
/**
* 单线程的线程池发送消息,避免阻塞主线程
*/
private static ExecutorService executor = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS,
new LinkedBlockingQueue<>(1024), Executors.defaultThreadFactory(), new ThreadPoolExecutor.DiscardPolicy());
public static void sendMessage(String title, String message, List<String> phones) {
if (StringUtils.isBlank(message)) {
return;
}
if (StringUtils.isBlank(title)) {
title = "异常通知";
}
final Map<String, Object> params = new HashMap<>(16);
params.put("title", title);
params.put("body", Collections.singletonList(message));
params.put("phones", phones);
final String fTitle = title;
final String fMessage = message;
final String fPhones = StringUtils.join(phones, ",");
Runnable runnable = new Runnable() {
@Override
public void run() {
CloseableHttpResponse res;
try {
HttpPost post = new HttpPost();
post.setURI(FEI_SHU_URL);
post.setEntity(new StringEntity(JSON.toJSON(params).toString(), ContentType.APPLICATION_JSON));
res = client.execute(post);
if (res == null || res.getEntity() == null) {
LOGGER.error("发送飞书消息失败: title:{}, message:{}, phones:{}", fTitle, fMessage, fPhones);
return;
}
if (res.getStatusLine() == null || res.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
LOGGER.error("发送飞书消息失败,title:{}, message:{}, phones:{}, res:{}", fTitle, fMessage, fPhones, EntityUtils.toString(res.getEntity()));
return;
}
} catch (IOException e) {
LOGGER.error("发送飞书消息失败: title:{}, message:{}, phones:{}", fTitle, fMessage, fPhones, e);
}
}
};
executor.submit(runnable);
}
public static void waitForFinish() throws InterruptedException {
executor.shutdown();
executor.awaitTermination(5, TimeUnit.MINUTES);
}
public static void main(String[] args) throws InterruptedException {
FeiShuUtil.sendMessage("测试", "hello wolrd", Arrays.asList("13426233960"));
FeiShuUtil.waitForFinish();
}
}
\ No newline at end of file
package com.secoo.so.suggest.util;
import com.github.stuxuhai.jpinyin.ChineseHelper;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class PinYinUtils {
/**
* 简体转换为繁体
*/
public static String convertToTraditionalChinese(String str) {
String tempStr = null;
try {
tempStr = ChineseHelper.convertToTraditionalChinese(str);
} catch (Exception e) {
tempStr = str;
log.error("convertToTraditionalChinese error", e);
}
return tempStr;
}
/**
* 繁体转换为简体
*/
public static String convertToSimplifiedChinese(String str) {
String tempStr = null;
try {
tempStr = ChineseHelper.convertToSimplifiedChinese(str);
} catch (Exception e) {
tempStr = str;
log.error("convertToSimplifiedChinese error", e);
}
return tempStr;
}
}
sqp.service.name=sqb4j
sqp.service.version=1.4
sqp.service.level=product
sqp.zookeeper.hosts=center1.secoo-inc.com:2181,center2.secoo-inc.com:2181,center3.secoo-inc.com:2181
image.delete.url=http://10.16.9.28:8080
image.update.url=http://10.16.8.44:8080
prefixFilterList=["https://", "http://", "dg", "d & g", "dolce&gabbana","dolce & gabbana", "\u675C\u5609\u73ED\u7EB3", "\u907F\u5B55", "\u60C5\u8DA3", "cucci", "\u4E52\u4E53\u7403", "cuccl", "gucii","tod's","iwc7"]
ManualFolder=/data/pssmaster/corpus_set/suggest_corpus/manual
SensitiveFolder=/data/pssmaster/corpus_set/suggest_corpus/sensitive
EuropeWordFolder=/data/pssmaster/corpus_set/suggest_corpus/europe_word
erp.read.url=jdbc:mysql://192.168.50.40:3306/secooErpDB?useUnicode=true&amp;characterEncoding=utf8&amp;noAccessToProcedureBodies=true&amp;zeroDateTimeBehavior=convertToNull&amp;allowMultiQueries=true
erp.read.user=so_Erp_R
erp.read.password=5RgzudyyFlApTmve
seo.read.url=jdbc:mysql://secooSeoDB.master.com:3307/secooSeoDB?useUnicode=true&amp;characterEncoding=utf8&amp;zeroDateTimeBehavior=convertToNull
seo.read.user=sem_Seo_W
seo.read.password=C2IiHfNKYpT1onsR
dw.read.url=jdbc:mysql://secooDataWarehouse.slave.com:3306/secooDataWarehouse?useUnicode=true&amp;characterEncoding=utf8&amp;zeroDateTimeBehavior=convertToNull
dw.read.user=Search_DataWar_R
dw.read.password=pY1P9zUj9x1M65ot5szo
\ No newline at end of file
sqp.service.name=sqb4j
sqp.service.version=1.4
sqp.service.level=product
sqp.zookeeper.hosts=center1.secoo-inc.com:2181,center2.secoo-inc.com:2181,center3.secoo-inc.com:2181
image.delete.url=http://192.168.70.141:8080
\ No newline at end of file
erp.read.url=jdbc:mysql://10.4.3.223:3306/secooErpDB?useUnicode=true&amp;characterEncoding=utf8&amp;noAccessToProcedureBodies=true&amp;zeroDateTimeBehavior=convertToNull&amp;allowMultiQueries=true
erp.read.user=3306_test
erp.read.password=iS6CXpYqgZ8Mhjui
seo.read.url=jdbc:mysql://10.4.3.223:3306/secooSeoDB?useUnicode=true&amp;characterEncoding=utf8&amp;zeroDateTimeBehavior=convertToNull
seo.read.user=SeoDB_test
seo.read.password=Cxkfq57huej0fTpK
\ No newline at end of file
log4j.rootLogger=INFO, FILE
# logger
log4j.logger.com.secoo.so.mysql.binlog=INFO
log4j.logger.org.apache.zookeeper=WARN
log4j.logger.com.alibaba.dubbo=WARN
log4j.appender.FILE=org.apache.log4j.RollingFileAppender
log4j.appender.FILE.File=logs/service.log
log4j.appender.FILE.MaxFileSize=100MB
log4j.appender.FILE.MaxBackupIndex=5
log4j.appender.FILE.Threshold=INFO
log4j.appender.FILE.layout=org.apache.log4j.PatternLayout
log4j.appender.FILE.layout.ConversionPattern=[%d{yyyy-MM-dd HH:mm:ss}]%.8t %-5p %c:%L %m%n
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment