Skip to content

M
matrix
Commit 118b51c0 by 房斌
Options

1监控路径配置2优化上报日志3去掉无关的配置

parent 60b52fbd
Showing with 212 additions and 150 deletions
matrix-gracefulshutdown/src/main/java/com/secoo/mall/dubbo/monitor/config/ConfigCenter.java
......@@ -35,26 +35,20 @@ public class ConfigCenter {
public static final Logger logger = LoggerFactory.getLogger(ConfigCenter.class);
//centers in dubbo 2.7
@Value("${admin.config-center:}")
public String configCenter;
@Value("${admin.registry.address:}")
public String registryAddress;
@Value("${admin.metadata-report.address:}")
public String metadataAddress;
@Value("${admin.metadata-report.cluster:false}")
public boolean cluster;
@Value("${admin.registry.group:dubbo}")
public String registryGroup;
@Value("${admin.config-center.group:dubbo}")
public String configCenterGroup;
@Value("${admin.metadata-report.group:dubbo}")
public String metadataGroup;
@Value("${admin.config-center.username:}")
public String username;
@Value("${admin.config-center.password:}")
public String password;
@Value("${admin.config-center.patch:}")
@Value("${admin.patch:}")
public String patch;
public URL configCenterUrl;
public URL registryUrl;
......
matrix-gracefulshutdown/src/main/java/com/secoo/mall/dubbo/monitor/dubbo/service/RegistryServerSync.java
......@@ -16,8 +16,12 @@
*/
package com.secoo.mall.dubbo.monitor.dubbo.service;
import com.alibaba.fastjson.JSON;
import com.secoo.mall.common.util.date.DateUtil;
import com.secoo.mall.common.util.string.StringUtil;
import com.secoo.mall.dubbo.monitor.dubbo.Constants.Constants;
import com.secoo.mall.dubbo.monitor.utils.CoderUtil;
import com.secoo.mall.dubbo.monitor.utils.Stack;
import com.secoo.mall.dubbo.monitor.utils.Tool;
import org.apache.dubbo.common.URL;
import org.apache.dubbo.common.logger.Logger;
......@@ -26,6 +30,8 @@ import org.apache.dubbo.common.utils.NetUtils;
import org.apache.dubbo.common.utils.StringUtils;
import org.apache.dubbo.registry.NotifyListener;
import org.apache.dubbo.registry.Registry;
import org.apache.dubbo.registry.support.AbstractRegistryFactory;
import org.apache.dubbo.registry.zookeeper.ZookeeperRegistry;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
......@@ -42,7 +48,7 @@ public class RegistryServerSync implements InitializingBean, DisposableBean, Not
this.registry=registry;
}
private static final Logger logger = LoggerFactory.getLogger(RegistryServerSync.class);
private final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(this.getClass());
private static final URL SUBSCRIBE = new URL(Constants.ADMIN_PROTOCOL, NetUtils.getLocalHost(), 0, "",
Constants.INTERFACE_KEY, Constants.ANY_VALUE,
......@@ -70,7 +76,7 @@ public class RegistryServerSync implements InitializingBean, DisposableBean, Not
private final ConcurrentMap<String, ConcurrentMap<String, Map<String, URL>>> registryCache = new ConcurrentHashMap<>();
// @Autowired
private Registry registry;
public Registry registry;
public ConcurrentMap<String, ConcurrentMap<String, Map<String, URL>>> getRegistryCache() {
......@@ -80,15 +86,9 @@ public class RegistryServerSync implements InitializingBean, DisposableBean, Not
@Override
public void afterPropertiesSet() throws Exception {
logger.info("Init Dubbo Admin Sync Cache...");
// FailbackRegistry parentRegister= (FailbackRegistry)registry;
// Set<URL> urls= parentRegister.getRegistered();
// for(URL u: urls){
// if(u.getParameter(Constants.CATEGORY_KEY).endsWith(Constants.PROVIDER_PROTOCOL)) {
// registry.subscribe(u, this);
// }
// }
registry.subscribe(SUBSCRIBE, this);
// registry.subscribe(SUBSCRIBE, this);
}
......@@ -103,6 +103,7 @@ public class RegistryServerSync implements InitializingBean, DisposableBean, Not
@Override
public void notify(List<URL> urls) {
logger.info("receive time url:"+urls.toString()+" time:" +new SimpleDateFormat("yyyy-MM-dd HH:mm:ss:SSS").format(new Date()));
if (urls == null || urls.isEmpty()) {
return;
}
......@@ -189,13 +190,31 @@ public class RegistryServerSync implements InitializingBean, DisposableBean, Not
logger.info("Init Dubbo Admin Sync Cache...");
// FailbackRegistry parentRegister= (FailbackRegistry)registry;
// Set<URL> urls= parentRegister.getRegistered();
// for(URL u: urls){
// if(u.getParameter(Constants.CATEGORY_KEY).endsWith(Constants.PROVIDER_PROTOCOL)) {
// registry.subscribe(u, this);
// }
// }
Collection<Registry> registries = AbstractRegistryFactory.getRegistries();
// logger.info("registries already"+JSON.toJSONString(registries));
Iterator<Registry> iterator = registries.iterator();
Set<URL> urls = null;
ZookeeperRegistry zookperRegster=null;
while (iterator.hasNext()) {
Registry r = iterator.next();
zookperRegster = (ZookeeperRegistry) r;
if (zookperRegster.getRegistered() != null && zookperRegster.getRegistered().size() > 0) {
urls = zookperRegster.getRegistered();
if (urls != null && urls.size() > 0) {
for(URL value: urls){
try {
logger.info("RegistryServerSync subscribe urls " + JSON.toJSONString(value));
registry.subscribe(value,this);
}catch (Exception e){
logger.info(" error :matirx-monitor offDubbo " + Stack.errInfo(e) + StringUtil.line());
}
}
}
}
break;
}
}
}
......
matrix-gracefulshutdown/src/main/java/com/secoo/mall/dubbo/monitor/utils/Stack.java 0 → 100644
package com.secoo.mall.dubbo.monitor.utils;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
public class Stack {
public static String errInfo(Exception e) {
StringWriter sw = null;
PrintWriter pw = null;
try {
sw = new StringWriter();
pw = new PrintWriter(sw);
// 将出错的栈信息输出到printWriter中
e.printStackTrace(pw);
pw.flush();
sw.flush();
} finally {
if (sw != null) {
try {
sw.close();
} catch (IOException e1) {
e1.printStackTrace();
}
}
if (pw != null) {
pw.close();
}
}
return sw.toString();
}
}
matrix-gracefulshutdown/src/main/java/com/secoo/mall/dubbo/service/DubboUpdata.java
......@@ -4,14 +4,19 @@ import com.alibaba.fastjson.JSON;
import com.secoo.mall.common.core.service.UpDatas;
import com.secoo.mall.common.util.log.LoggerUtil;
import com.secoo.mall.common.core.bean.gracefulshowtdownBean.ExecutorDetails;
import com.secoo.mall.common.util.string.StringUtil;
import org.apache.dubbo.common.URL;
import org.apache.dubbo.common.extension.ExtensionLoader;
import org.apache.dubbo.remoting.zookeeper.ZookeeperClient;
import org.apache.dubbo.remoting.zookeeper.ZookeeperTransporter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
@Component
public class DubboUpdata implements UpDatas<ExecutorDetails> {
private final Logger logger = LoggerFactory.getLogger(this.getClass());
String path;
URL url;
public URL getUrl() {
......@@ -22,15 +27,25 @@ public class DubboUpdata implements UpDatas<ExecutorDetails> {
this.url = url;
}
public String getPath() {
return path;
}
public void setPath(String path) {
this.path = path;
}
@Override
public boolean upData(ExecutorDetails obj) {
ZookeeperTransporter zookeeperTransporter = ExtensionLoader.getExtensionLoader(ZookeeperTransporter.class).getAdaptiveExtension();
ZookeeperClient cliet = zookeeperTransporter.connect(url);
// String monitor=url.getPath()+"/"+obj.getName()+"/"+obj.getIp();
// String monitor=url.getPath();
if(StringUtil.isEmpty(path)){
path="/monitorZ";
}
String monitor=path+"/"+obj.getName()+"/"+obj.getIp();
try {
String monitor = "/monitorZ/" + obj.getName() + "/" + obj.getIp();
ZookeeperClient cliet = zookeeperTransporter.connect(url);
// String monitor = "/monitorZ/" + obj.getName() + "/" + obj.getIp();
logger.info("DubboUpdata sendData: path:"+monitor+" data:"+ JSON.toJSONString(obj));
cliet.create(monitor, JSON.toJSONString(obj), true);
} catch (Exception e) {
LoggerUtil.info("Exception e:" ,e);
......
matrix-gracefulshutdown/src/main/java/com/secoo/mall/dubbo/service/impl/DubboCustomerDownHock.java
package com.secoo.mall.dubbo.service.impl;
import com.secoo.mall.common.core.bean.gracefulshowtdownBean.ExecutorDetail;
import com.secoo.mall.common.util.date.DateUtil;
import com.secoo.mall.common.util.log.LoggerUtil;
import com.secoo.mall.common.util.string.StringUtil;
import com.secoo.mall.common.core.bean.gracefulshowtdownBean.ExecutorDetail;
import com.secoo.mall.dubbo.monitor.dubbo.model.domain.Provider;
import com.secoo.mall.dubbo.monitor.dubbo.service.ProviderService;
import com.secoo.mall.dubbo.monitor.utils.Stack;
import com.secoo.mall.dubbo.service.AbstractStop;
import org.apache.commons.lang3.StringUtils;
import org.apache.dubbo.common.URL;
import org.apache.dubbo.common.extension.ExtensionLoader;
import org.apache.dubbo.common.utils.NetUtils;
import org.apache.dubbo.config.DubboShutdownHook;
import org.apache.dubbo.registry.Registry;
import org.apache.dubbo.registry.support.AbstractRegistryFactory;
import org.apache.dubbo.registry.zookeeper.ZookeeperRegistry;
import org.apache.dubbo.remoting.zookeeper.curator.CuratorZookeeperClient;
import org.apache.dubbo.rpc.Protocol;
import org.apache.dubbo.rpc.model.ApplicationModel;
import org.springframework.beans.factory.annotation.Autowired;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.lang.reflect.Field;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.*;
public class DubboCustomerDownHock extends AbstractStop {
//@Autowired
private ProviderService providerService;
private ProviderService providerService;
public DubboCustomerDownHock(ProviderService providerService ) {
public DubboCustomerDownHock(ProviderService providerService) {
this.providerService = providerService;
}
private final Logger logger = LoggerFactory.getLogger(this.getClass());
@Override
public ExecutorDetail stop() {
ExecutorDetail detail = new ExecutorDetail();
detail.setBeginTime(DateUtil.getDateTime());
detail.setServiceName("dubboDownHock");
detail.setServiceName("dubboDownHock1001");
List<String> str = new ArrayList<String>();
detail.setDetail(str);
detail.setCode(0);
String ip = NetUtils.getIpByHost(NetUtils.getLocalAddress().getHostName());
String name = ApplicationModel.getApplication();
try {
//step1 register销毁
String ip = NetUtils.getIpByHost(NetUtils.getLocalAddress().getHostName());
String name = ApplicationModel.getApplication();
List<Provider> serviceDTOS = providerService.getServiceDTOSByQuery("ip", ip, "");
if (StringUtil.isNoneEmpty(name) && serviceDTOS != null && serviceDTOS.size() > 0) {
serviceDTOS = providerService.findProviderUrlByGivenApplication(name, serviceDTOS);
}
Iterator<Provider> provides = serviceDTOS.iterator();
while (provides.hasNext()) {
Provider provider = provides.next();
try {
if (StringUtils.isNotEmpty(String.valueOf(provider.getHash()))) {
providerService.deleteStaticProvider(String.valueOf(provider.getHash()));
LoggerUtil.info("delete2 provider url:" + provider.getService() + " time:" + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss:SSS").format(new Date()));
Collection<Registry> registries = AbstractRegistryFactory.getRegistries();
Set<URL> urls = null;
List<URL> checks=null;
ZookeeperRegistry zookperRegster = null;
if (registries != null && registries.size() > 0) {
Iterator<Registry> iterator = registries.iterator();
while (iterator.hasNext()) {
Registry r = iterator.next();
zookperRegster = (ZookeeperRegistry) r;
if (zookperRegster.getRegistered() != null && zookperRegster.getRegistered().size() > 0) {
urls = zookperRegster.getRegistered();
if (urls != null && urls.size() > 0) {
checks=new ArrayList<URL>();
for (URL value : urls) {
checks.add(value);
try {
str.add("provider url executer unregister before:" + value.getServiceInterface() + " time:" + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss:SSS").format(new Date()) +StringUtil.line());
zookperRegster.unregister((URL) value);
LoggerUtil.info("provider url unregister success :" + value.getServiceInterface() + " time:" + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss:SSS").format(new Date()) +StringUtil.line());
str.add("delete provider url success:" + value.getServiceInterface() + " time:" + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss:SSS").format(new Date()) +StringUtil.line());
} catch (Exception e) {
LoggerUtil.error("matirx-monitor deleteService error:", e);
str.add("matirx-monitor delete Service error service:"+value.getServiceInterface() + StringUtil.line() +Stack.errInfo(e)+StringUtil.line());
}
}
}else{
str.add("warn:matrix-monitor no dubbo registered for delete ip " + ip + " name :" +name+ StringUtil.line());
detail.setCode(-1);
}
}
} catch (Exception e) {
LoggerUtil.error("matirx-monitor offDubbo deleteStaticProvider error:", e);
break;
}
}
Thread.sleep(6000);//zookper 目前最大超时时间
//step3 check检查
Iterator<Provider> iteratorCheck = serviceDTOS.iterator();
while (iteratorCheck.hasNext()) {
Provider obj = iteratorCheck.next();
if (StringUtil.isNotEmpty(obj.getAddress()) && StringUtil.isNotEmpty(name)) {
List<String> check = providerService.findServicesByAddressAndName(obj.getAddress(), name);
if (check != null && check.size() > 0) {
str.add("warn:matrix-monitor2 off DubboService check service failure" + obj.getUrl() + " already on line after off " + StringUtil.line());
detail.setCode(-1);
if(checks!=null&&checks.size()>0) {
//等待
CuratorZookeeperClient client = (CuratorZookeeperClient) getFieldValueByFieldName("zkClient", zookperRegster);
int sessionExpireMs = client.getUrl().getParameter("zk.session.expire", 10000);
Thread.sleep(sessionExpireMs);//zookper 目前最大超时时间
for( URL url :checks) {
if (StringUtil.isNotEmpty(url.getAddress()) && StringUtil.isNotEmpty(name)) {
List<String> check = null;
try {
check = providerService.findServicesByAddressAndName(url.getAddress(), name);
} catch (Exception e) {
detail.setCode(-1);
str.add(" error :matirx-monitor checko Dubbo service " + StringUtil.line()+ Stack.errInfo(e) + StringUtil.line());
}
if (check != null && check.size() > 0) {
str.add("warn:matrix-monitor check service failure " + url.getServiceKey() + " reason zk notice failed " + StringUtil.line());
detail.setCode(-1);
}else{
str.add("matrix-monitor dubbo check service ok " + url.getServiceKey() + StringUtil.line());
}
}
}
}else{
str.add("warn:matrix-monitor no urls check ip：" + ip + " name :"+name + StringUtil.line());
logger.info("warn:matrix-monitor no urls check ip：" + ip + " name :"+name + StringUtil.line());
}
} catch (Exception e) {
} catch (
Exception e) {
detail.setCode(-1);
str.add(" error :matirx-monitor offDubbo " + e.toString() + StringUtil.line());
}finally {
str.add(" error :matirx-monitor offDubbo " + Stack.errInfo(e) + StringUtil.line());
} finally {
DubboShutdownHook.getDubboShutdownHook().doDestroy();
}
detail.setEndTime(DateUtil.getDateTime());
logger.info("GracefullyShoutDown dubbo end execute:name:" + name + " ip:" + ip + " time:" + DateUtil.getDateTime());
return detail;
}
......@@ -109,4 +143,24 @@ public class DubboCustomerDownHock extends AbstractStop {
}
/**
* 根据属性名获取属性值
*
* @param fieldName
* @param object
* @return
*/
private Object getFieldValueByFieldName(String fieldName, Object object) {
try {
Field field = object.getClass().getDeclaredField(fieldName);
//设置对象的访问权限，保证对private的属性的访问
field.setAccessible(true);
return (Object) field.get(object);
} catch (Exception e) {
return null;
}
}
}
matrix-gracefulshutdown/src/main/java/com/secoo/mall/dubbo/service/impl/ServletConnectShoutDownHock.java
......@@ -52,6 +52,8 @@ public class ServletConnectShoutDownHock extends AbstractStop {
detail.setCode(-1);
str.add("warn:Tomcat thread pool did not terminate already wait 20s" + StringUtil.line());
}
}else{
str.add("warn:Tomcat thread pool gracefully shutdown already" + StringUtil.line());
}
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
......
matrix-gracefulshutdown/src/main/java/com/secoo/mall/dubbo/spring/boot/autoconfigure/GracefullyShoutDown.java
......@@ -3,16 +3,24 @@ package com.secoo.mall.dubbo.spring.boot.autoconfigure;
import com.alibaba.fastjson.JSON;
import com.secoo.mall.common.core.service.StopService;
import com.secoo.mall.common.core.service.UpDatas;
import com.secoo.mall.common.util.date.DateUtil;
import com.secoo.mall.common.util.log.LoggerUtil;
import com.secoo.mall.common.core.bean.gracefulshowtdownBean.ExecutorDetail;
import com.secoo.mall.common.core.bean.gracefulshowtdownBean.ExecutorDetails;
import com.secoo.mall.common.util.string.StringUtil;
import com.secoo.mall.dubbo.monitor.dubbo.service.ProviderService;
import com.secoo.mall.dubbo.monitor.dubbo.service.RegistryServerSync;
import com.secoo.mall.dubbo.monitor.utils.Stack;
import com.secoo.mall.dubbo.service.impl.ServletConnectShoutDownHock;
import org.apache.catalina.connector.Connector;
import org.apache.dubbo.common.URL;
import org.apache.dubbo.common.extension.ExtensionLoader;
import org.apache.dubbo.common.utils.NetUtils;
import org.apache.dubbo.config.DubboShutdownHook;
import org.apache.dubbo.config.spring.extension.SpringExtensionFactory;
import org.apache.dubbo.registry.Registry;
import org.apache.dubbo.registry.support.AbstractRegistryFactory;
import org.apache.dubbo.registry.zookeeper.ZookeeperRegistry;
import org.apache.dubbo.rpc.Protocol;
import org.apache.dubbo.rpc.model.ApplicationModel;
import org.slf4j.Logger;
......@@ -63,15 +71,10 @@ public class GracefullyShoutDown implements CommandLineRunner, ApplicationListen
Class clz = SpringExtensionFactory.class;
multicaster.removeApplicationListener((ApplicationListener) getPrivateConst("SHUTDOWN_HOOK_LISTENER"));
}
// //上报数据，异步上报 超时返回
// ResultTransportData data=new ResultTransportData();
// String ip = NetUtils.getIpByHost(NetUtils.getLocalAddress().getHostName());
// String name = ApplicationModel.getApplication();
// data.setIp(ip);
// data.setName(name);
// data.setT("hello,nihao");
// dubboUpdata.upData(data);
//test-------------------------------------> begin
ExecutorDetails result = new ExecutorDetails();
result.setCode(0);
......@@ -81,7 +84,7 @@ public class GracefullyShoutDown implements CommandLineRunner, ApplicationListen
String name = ApplicationModel.getApplication();
result.setIp(ip);
result.setName(name);
//
try {
Map<String, StopService> map = this.context.getBeansOfType(StopService.class);
if (map != null && map.size() > 0) {
......@@ -115,7 +118,7 @@ public class GracefullyShoutDown implements CommandLineRunner, ApplicationListen
//test------------------------------------->
}
}
public static Object getPrivateConst(String field) {
......@@ -146,6 +149,7 @@ public class GracefullyShoutDown implements CommandLineRunner, ApplicationListen
@Override
public void onApplicationEvent(ContextClosedEvent contextClosedEvent) {
ExecutorDetails result = new ExecutorDetails();
result.setCode(0);
List<ExecutorDetail> details = new ArrayList<ExecutorDetail>();
......@@ -154,6 +158,8 @@ public class GracefullyShoutDown implements CommandLineRunner, ApplicationListen
String name = ApplicationModel.getApplication();
result.setIp(ip);
result.setName(name);
logger.info("GracefullyShoutDown execute:name:"+name+" ip:"+ip+" time:"+ DateUtil.getDateTime());
try {
Map<String, StopService> map = contextClosedEvent.getApplicationContext().getBeansOfType(StopService.class);
......@@ -184,6 +190,7 @@ public class GracefullyShoutDown implements CommandLineRunner, ApplicationListen
} catch (Exception e) {
LoggerUtil.error("matrix.GracefullyShoutDown update .error data:" + JSON.toJSONString(result), e);
}
logger.info("GracefullyShoutDown execute end :name:"+name+" ip:"+ip+" time:"+ DateUtil.getDateTime());
}
@Override
......
matrix-gracefulshutdown/src/main/java/com/secoo/mall/dubbo/spring/boot/autoconfigure/MatrixGracefulShutDownAutoConfiguration.java
......@@ -39,8 +39,8 @@ public class MatrixGracefulShutDownAutoConfiguration {
UpDatas dubboUpdata( ConfigCenter configCenter){
DubboUpdata transport= new DubboUpdata();
URL url =configCenter.formUrl(configCenter.registryAddress, configCenter.registryGroup, configCenter.username, configCenter.password);
URL newPath= url.setPath(configCenter.patch);
transport.setUrl(newPath);
transport.setUrl(url);
transport.setPath(configCenter.patch);
return transport;
}
......
matrix-protocol/matrix-protocol-dubbo-core/src/main/java/com/secoo/mall/dubbo/service/ServletConnectShoutDownHock.java deleted 100644 → 0
package com.secoo.mall.dubbo.service;
import com.secoo.mall.common.core.service.AbstractStop;
import com.secoo.mall.common.util.date.DateUtil;
import com.secoo.mall.common.util.log.LoggerUtil;
import org.apache.catalina.connector.Connector;
import org.springframework.stereotype.Component;
import java.util.concurrent.Executor;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
//springboot 容器关闭
@Component
public class ServletConnectShoutDownHock extends AbstractStop {
private volatile Connector connector;
public Connector getConnector() {
return connector;
}
public void setConnector(Connector connector) {
this.connector = connector;
}
@Override
public StringBuilder stop() {
StringBuilder sb = new StringBuilder();
this.connector.pause();
Executor executor = this.connector.getProtocolHandler().getExecutor();
if (executor instanceof ThreadPoolExecutor) {
try {
ThreadPoolExecutor threadPoolExecutor = (ThreadPoolExecutor) executor;
threadPoolExecutor.shutdown();
LoggerUtil.info("apache connector executor shutdown=============time:" + DateUtil.getDateTime());
//TODO 超时时间
if (!threadPoolExecutor.awaitTermination(10, TimeUnit.SECONDS)) {
LoggerUtil.info("Tomcat thread pool did not shut down gracefully within "
+ "10" + " seconds. Proceeding with forceful shutdown");
threadPoolExecutor.shutdownNow();
if (!threadPoolExecutor.awaitTermination(10, TimeUnit.SECONDS)) {
LoggerUtil.info("Tomcat thread pool did not terminate");
sb.append("Tomcat thread pool did not terminate");
}
}
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
sb.append(ex.getMessage());
}
}
return sb;
}
@Override
public Integer getHandleTypeOrder() {
return 1;
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment