From 96639ccd37ebfa90221a7d2ba2e4a27878e861d2 Mon Sep 17 00:00:00 2001 From: "583641232@qq.com" <583641232@qq.com> Date: Wed, 22 Jan 2025 12:17:04 +0800 Subject: [PATCH] :sparkles: v3 --- .../alog/common/model/ActionLogMessage.java | 125 ++--------------- .../alog/common/model/TracerData.java | 5 +- .../alog/nacos/NacosListener.java | 2 +- example/pom.xml | 45 +----- .../clientdemo/aspectj/ActionLogAspect.java | 12 +- .../clientdemo/custom/CustomConfigurator.java | 100 +++++++------- .../alog/clientdemo/custom/Starter.java | 20 +-- .../demo/controller/DemoUserController.java | 45 +++--- .../alog/clientdemo/demo/domain/DemoUser.java | 5 +- .../alog/clientdemo/demo/domain/R.java | 130 ------------------ .../service/impl/DemoUserServiceImpl.java | 4 - .../alog/clientdemo/utils/BeanCopyUtils.java | 30 +--- .../alog/clientdemo/utils/StringUtils.java | 5 +- example/src/main/resources/application.yml | 5 +- worker/pom.xml | 6 + .../alog/worker/store/ActionLogToDbStore.java | 116 ++++++++++++++++ worker/src/main/resources/application.yml | 7 + worker/src/main/resources/jlog.sql | 34 ++++- 18 files changed, 270 insertions(+), 426 deletions(-) delete mode 100644 example/src/main/java/com/inscloudtech/alog/clientdemo/demo/domain/R.java create mode 100644 worker/src/main/java/com/inscloudtech/alog/worker/store/ActionLogToDbStore.java diff --git a/common/src/main/java/com/inscloudtech/alog/common/model/ActionLogMessage.java b/common/src/main/java/com/inscloudtech/alog/common/model/ActionLogMessage.java index 6f0405f..fe3229a 100644 --- a/common/src/main/java/com/inscloudtech/alog/common/model/ActionLogMessage.java +++ b/common/src/main/java/com/inscloudtech/alog/common/model/ActionLogMessage.java @@ -7,43 +7,19 @@ import java.util.Map; * className：RunLogMessage * description： * - * @author wuweifeng + * @author * @version 1.0.0 */ public class ActionLogMessage { - /** - * tracerId - */ - private long tracerId; /** * 时间创建时间 */ private long createTime; - /** - * 日志内容 - */ - private Object content; - /** - * info、error - */ - private String logLevel; - /** - * 类名 - */ - private String className; - /** - * 方法名 - */ - private String methodName; - /** - * 线程名 - */ - private String threadName; /** * 日志主键 */ - private Long operId; + private long logId; /** * 操作模块 */ @@ -52,10 +28,7 @@ public class ActionLogMessage { * 业务类型（0其它 1新增 2修改 3删除） */ private Integer businessType; - /** - * 业务类型数组 - */ - private Integer[] businessTypes; + /** * 请求方法 */ @@ -150,14 +123,6 @@ public class ActionLogMessage { this.operUserName = operUserName; } - public Long getOperId() { - return operId; - } - - public void setOperId(Long operId) { - this.operId = operId; - } - public String getBusinessName() { return businessName; } @@ -174,14 +139,6 @@ public class ActionLogMessage { this.businessType = businessType; } - public Integer[] getBusinessTypes() { - return businessTypes; - } - - public void setBusinessTypes(Integer[] businessTypes) { - this.businessTypes = businessTypes; - } - public String getMethod() { return method; } @@ -310,17 +267,14 @@ public class ActionLogMessage { this.businessId = businessId; } - /** - * 标签map - */ - private Map tagMap; - public long getTracerId() { - return tracerId; + + public long getLogId() { + return logId; } - public void setTracerId(long tracerId) { - this.tracerId = tracerId; + public void setLogId(long logId) { + this.logId = logId; } public long getCreateTime() { @@ -331,68 +285,5 @@ public class ActionLogMessage { this.createTime = createTime; } - public Object getContent() { - return content; - } - public void setContent(Object content) { - this.content = content; - } - - public String getLogLevel() { - return logLevel; - } - - public void setLogLevel(String logLevel) { - this.logLevel = logLevel; - } - - public String getClassName() { - return className; - } - - public void setClassName(String className) { - this.className = className; - } - - public String getMethodName() { - return methodName; - } - - public void setMethodName(String methodName) { - this.methodName = methodName; - } - - public String getThreadName() { - return threadName; - } - - public void setThreadName(String threadName) { - this.threadName = threadName; - } - - public Map getTagMap() { - return tagMap; - } - - public void setTagMap(Map tagMap) { - this.tagMap = tagMap; - } - - - - - @Override - public String toString() { - return "RunLogMessage{" + - "tracerId=" + tracerId + - ", createTime=" + createTime + - ", content=" + content + - ", logLevel='" + logLevel + '\'' + - ", className='" + className + '\'' + - ", methodName='" + methodName + '\'' + - ", threadName='" + threadName + '\'' + - ", tagMap=" + tagMap + - '}'; - } } diff --git a/common/src/main/java/com/inscloudtech/alog/common/model/TracerData.java b/common/src/main/java/com/inscloudtech/alog/common/model/TracerData.java index 6da95da..f562adb 100644 --- a/common/src/main/java/com/inscloudtech/alog/common/model/TracerData.java +++ b/common/src/main/java/com/inscloudtech/alog/common/model/TracerData.java @@ -77,10 +77,7 @@ public class TracerData implements Serializable { @Override public String toString() { return "TracerData{" + - "type=" + type + - ", tracerBeanList=" + tracerBeanList + - ", tempLogs=" + tempLogs + - ", address=" + address + + "actionLogs=" + actionLogs + '}'; } } diff --git a/config/config-nacos/src/main/java/com/inscloudtech/alog/nacos/NacosListener.java b/config/config-nacos/src/main/java/com/inscloudtech/alog/nacos/NacosListener.java index 50c2207..37d56f0 100644 --- a/config/config-nacos/src/main/java/com/inscloudtech/alog/nacos/NacosListener.java +++ b/config/config-nacos/src/main/java/com/inscloudtech/alog/nacos/NacosListener.java @@ -16,7 +16,7 @@ import java.io.*; import java.util.Set; -import static com.inscloudtech.platform.jlog.nacos.NacosConfigurator.*; +import static com.inscloudtech.alog.nacos.NacosConfigurator.*; /** diff --git a/example/pom.xml b/example/pom.xml index f497472..0ddfc3b 100644 --- a/example/pom.xml +++ b/example/pom.xml @@ -46,55 +46,18 @@ spring-boot-configuration-processor true - - org.projectlombok - lombok - 1.18.20 - test - com.baomidou mybatis-plus-boot-starter 3.4.3 + com.mysql mysql-connector-j 8.0.33 - - - - - - - - - - org.springframework.boot @@ -107,12 +70,6 @@ - - org.projectlombok - lombok - 1.18.20 - compile - diff --git a/example/src/main/java/com/inscloudtech/alog/clientdemo/aspectj/ActionLogAspect.java b/example/src/main/java/com/inscloudtech/alog/clientdemo/aspectj/ActionLogAspect.java index d820879..47c7129 100644 --- a/example/src/main/java/com/inscloudtech/alog/clientdemo/aspectj/ActionLogAspect.java +++ b/example/src/main/java/com/inscloudtech/alog/clientdemo/aspectj/ActionLogAspect.java @@ -11,6 +11,7 @@ import com.inscloudtech.alog.common.model.ActionLogMessage; import com.inscloudtech.alog.client.udp.UdpSender; import com.inscloudtech.alog.common.annotation.ActionLog; import com.inscloudtech.alog.common.utils.FastJsonUtils; +import com.inscloudtech.alog.common.utils.IdWorker; import org.aspectj.lang.JoinPoint; import org.aspectj.lang.ProceedingJoinPoint; import org.aspectj.lang.annotation.Around; @@ -157,8 +158,8 @@ public class ActionLogAspect { actionLogMessage.setRequestMethod(ServletUtils.getRequest().getMethod()); // 处理设置注解上的参数 getControllerMethodDescription(joinPoint, actionLog, actionLogMessage, jsonResult); - long tracerId = TracerHolder.getTracerId(); - actionLogMessage.setTracerId(tracerId); + long tracerId = IdWorker.nextId(); + actionLogMessage.setLogId(tracerId); actionLogMessage.setCreateTime(System.currentTimeMillis()); UdpSender.offerActionLogger(actionLogMessage); } catch (Exception exp) { @@ -167,6 +168,13 @@ public class ActionLogAspect { exp.printStackTrace(); } } + + public static void main(String[] args) { + long l = IdWorker.nextId(); + System.out.println((l+"").length()); + } + + /** * 获取注解中对方法的描述信息 用于Controller层注解 */ diff --git a/example/src/main/java/com/inscloudtech/alog/clientdemo/custom/CustomConfigurator.java b/example/src/main/java/com/inscloudtech/alog/clientdemo/custom/CustomConfigurator.java index 0ff8967..f2c117b 100644 --- a/example/src/main/java/com/inscloudtech/alog/clientdemo/custom/CustomConfigurator.java +++ b/example/src/main/java/com/inscloudtech/alog/clientdemo/custom/CustomConfigurator.java @@ -1,50 +1,50 @@ -package com.inscloudtech.alog.clientdemo.custom; - -import com.inscloudtech.alog.core.ConfiguratorFactory; -import com.inscloudtech.alog.core.FileConfigurator; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import java.io.IOException; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -/** - * @author tangbohu - * @version 1.0.0 - * @desc 基于redis实现自定义配置器示例，主要用于重写获取workers实现（毕竟是动态的，文件配置器不好处理） - * @ClassName CustomConfigurator.java - * @createTime 2022年03月21日 11:37:00 - */ -@Component -public class CustomConfigurator extends FileConfigurator { - private CustomConfigurator() throws IOException { - super(); - /** - * 由于配置器构建是static方法，在spring注入之前已经完成了 这里自定义配置器需要手动覆盖 - */ - ConfiguratorFactory.cover(this); - } - - - @Autowired - private RedisUtil redisUtil; - - - /** - * 获取list - */ - @Override - public List getList(String key) { - Set set = redisUtil.getMembers(key); - return set.stream().map(v->(String)v).collect(Collectors.toList()); - } - - - @Override - public String getType() { - return "custom-redis"; - } - -} +//package com.inscloudtech.alog.clientdemo.custom; +// +//import com.inscloudtech.alog.core.ConfiguratorFactory; +//import com.inscloudtech.alog.core.FileConfigurator; +//import org.springframework.beans.factory.annotation.Autowired; +//import org.springframework.stereotype.Component; +// +//import java.io.IOException; +//import java.util.List; +//import java.util.Set; +//import java.util.stream.Collectors; +// +///** +// * @author tangbohu +// * @version 1.0.0 +// * @desc 基于redis实现自定义配置器示例，主要用于重写获取workers实现（毕竟是动态的，文件配置器不好处理） +// * @ClassName CustomConfigurator.java +// * @createTime 2022年03月21日 11:37:00 +// */ +//@Component +//public class CustomConfigurator extends FileConfigurator { +// private CustomConfigurator() throws IOException { +// super(); +// /** +// * 由于配置器构建是static方法，在spring注入之前已经完成了 这里自定义配置器需要手动覆盖 +// */ +// ConfiguratorFactory.cover(this); +// } +// +// +// @Autowired +// private RedisUtil redisUtil; +// +// +// /** +// * 获取list +// */ +// @Override +// public List getList(String key) { +// Set set = redisUtil.getMembers(key); +// return set.stream().map(v->(String)v).collect(Collectors.toList()); +// } +// +// +// @Override +// public String getType() { +// return "custom-redis"; +// } +// +//} diff --git a/example/src/main/java/com/inscloudtech/alog/clientdemo/custom/Starter.java b/example/src/main/java/com/inscloudtech/alog/clientdemo/custom/Starter.java index bc2ceb2..f8a863b 100644 --- a/example/src/main/java/com/inscloudtech/alog/clientdemo/custom/Starter.java +++ b/example/src/main/java/com/inscloudtech/alog/clientdemo/custom/Starter.java @@ -55,14 +55,14 @@ public class Starter { tracerClientStarter.startPipeline(); } - @Bean - public FilterRegistrationBean urlFilter() { - FilterRegistrationBean registration = new FilterRegistrationBean(); - HttpFilter userFilter = new HttpFilter(); - registration.setFilter(userFilter); - registration.addUrlPatterns("/*"); - registration.setName("UserTraceFilter"); - registration.setOrder(1); - return registration; - } +// @Bean +// public FilterRegistrationBean urlFilter() { +// FilterRegistrationBean registration = new FilterRegistrationBean(); +// HttpFilter userFilter = new HttpFilter(); +// registration.setFilter(userFilter); +// registration.addUrlPatterns("/*"); +// registration.setName("UserTraceFilter"); +// registration.setOrder(1); +// return registration; +// } } diff --git a/example/src/main/java/com/inscloudtech/alog/clientdemo/demo/controller/DemoUserController.java b/example/src/main/java/com/inscloudtech/alog/clientdemo/demo/controller/DemoUserController.java index 033eee3..66a5e19 100644 --- a/example/src/main/java/com/inscloudtech/alog/clientdemo/demo/controller/DemoUserController.java +++ b/example/src/main/java/com/inscloudtech/alog/clientdemo/demo/controller/DemoUserController.java @@ -6,19 +6,19 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.toolkit.Wrappers; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.inscloudtech.alog.clientdemo.demo.domain.DemoUser; -import com.inscloudtech.alog.clientdemo.demo.domain.R; import com.inscloudtech.alog.clientdemo.demo.mapper.DemoUserMapper; import com.inscloudtech.alog.clientdemo.demo.service.DemoUserService; +import com.inscloudtech.alog.clientdemo.web.Response; import com.inscloudtech.alog.common.annotation.ActionLog; import com.inscloudtech.alog.common.enums.BusinessType; import com.inscloudtech.alog.common.model.TracerBean; import com.inscloudtech.alog.common.utils.FastJsonUtils; import com.inscloudtech.alog.core.Configurator; import com.inscloudtech.alog.core.ConfiguratorFactory; -import lombok.RequiredArgsConstructor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; import javax.servlet.http.HttpServletResponse; @@ -31,11 +31,11 @@ import javax.servlet.http.HttpServletResponse; * @date 2023-07-22 */ @RestController -@RequiredArgsConstructor @RequestMapping("/demoUser") public class DemoUserController { - private final DemoUserService demoUserService; + @Autowired + DemoUserService demoUserService; /** * 分页查询 @@ -45,57 +45,51 @@ public class DemoUserController { */ @GetMapping("/page" ) @ActionLog(businessName = "用户管理", businessType = BusinessType.SELECT) - public R page(Page page, DemoUser DemoUser) { - return R.ok(demoUserService.page(page, Wrappers.query(DemoUser))); + public Response page(Page page, DemoUser DemoUser) { + return new Response(demoUserService.page(page, Wrappers.query(DemoUser))); } /** * 通过id查询示例用户 * @param id id - * @return R + * @return Response */ @GetMapping("/{id}" ) - public R getById(@PathVariable("id" ) Long id) { - return R.ok(demoUserService.getById(id)); + public Response getById(@PathVariable("id" ) Long id) { + return new Response(demoUserService.getById(id)); } /** * 新增示例用户 * @param - * @return R + * @return Response */ @ActionLog(businessName = "用户管理", businessType = BusinessType.INSERT) @PostMapping - public R save(@RequestBody DemoUser demoUser) { - return R.ok(demoUserService.save(demoUser)); - } - - public static void main(String[] args) { - DemoUser a = new DemoUser(); - a.setNickName("演示用户"); - System.out.println("FastJsonUtils.convertObjectToJSON(a) = " + FastJsonUtils.convertObjectToJSON(a)); + public Response save(@RequestBody DemoUser demoUser) { + return new Response(demoUserService.save(demoUser)); } /** * 修改示例用户 * @param - * @return R + * @return Response */ @ActionLog(businessName = "用户管理",mapperClass = DemoUserMapper.class, businessType = BusinessType.UPDATE) @PutMapping - public R updateById(@RequestBody DemoUser demoUser) { - return R.ok(demoUserService.updateById(demoUser)); + public Response updateById(@RequestBody DemoUser demoUser) { + return new Response(demoUserService.updateById(demoUser)); } /** * 通过id删除示例用户 * @param id id - * @return R + * @return Response */ @DeleteMapping("/{id}" ) - public R removeById(@PathVariable Long id) { - return R.ok(demoUserService.removeById(id)); + public Response removeById(@PathVariable Long id) { + return new Response(demoUserService.removeById(id)); } /** @@ -103,8 +97,9 @@ public class DemoUserController { */ @ActionLog(businessName = "用户管理",mapperClass = DemoUserMapper.class, businessType = BusinessType.CLEAN) @PostMapping("/clean") - public void clean() { + public Response clean() { demoUserService.remove(new LambdaQueryWrapper<>()); + return new Response(0,"清空数据成功"); } /** diff --git a/example/src/main/java/com/inscloudtech/alog/clientdemo/demo/domain/DemoUser.java b/example/src/main/java/com/inscloudtech/alog/clientdemo/demo/domain/DemoUser.java index 5dec191..c6fa94c 100644 --- a/example/src/main/java/com/inscloudtech/alog/clientdemo/demo/domain/DemoUser.java +++ b/example/src/main/java/com/inscloudtech/alog/clientdemo/demo/domain/DemoUser.java @@ -2,7 +2,8 @@ package com.inscloudtech.alog.clientdemo.demo.domain; import com.baomidou.mybatisplus.annotation.TableId; -import lombok.Data; + +import java.io.Serializable; /** * 示例用户 stock @@ -10,7 +11,7 @@ import lombok.Data; * @date 2023-07-27 */ -public class DemoUser { +public class DemoUser implements Serializable { private static final long serialVersionUID=1L; /** diff --git a/example/src/main/java/com/inscloudtech/alog/clientdemo/demo/domain/R.java b/example/src/main/java/com/inscloudtech/alog/clientdemo/demo/domain/R.java deleted file mode 100644 index 531a1d8..0000000 --- a/example/src/main/java/com/inscloudtech/alog/clientdemo/demo/domain/R.java +++ /dev/null @@ -1,130 +0,0 @@ -package com.inscloudtech.alog.clientdemo.demo.domain; - -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.io.Serializable; - -/** - * 响应信息主体 - * - * @author Lion Li - */ - -@NoArgsConstructor -public class R implements Serializable { - private static final long serialVersionUID = 1L; - - /** - * 成功 - */ - public static final int SUCCESS = 200; - - /** - * 失败 - */ - public static final int FAIL = 500; - - private int code; - - private String msg; - - private T data; - - public int getCode() { - return code; - } - - public void setCode(int code) { - this.code = code; - } - - public String getMsg() { - return msg; - } - - public void setMsg(String msg) { - this.msg = msg; - } - - public T getData() { - return data; - } - - public void setData(T data) { - this.data = data; - } - - public static R ok() { - return restResult(null, SUCCESS, "操作成功"); - } - - public static R ok(T data) { - return restResult(data, SUCCESS, "操作成功"); - } - - public static R ok(String msg) { - return restResult(null, SUCCESS, msg); - } - - public static R ok(String msg, T data) { - return restResult(data, SUCCESS, msg); - } - - public static R fail() { - return restResult(null, FAIL, "操作失败"); - } - - public static R fail(String msg) { - return restResult(null, FAIL, msg); - } - - public static R fail(T data) { - return restResult(data, FAIL, "操作失败"); - } - - public static R fail(String msg, T data) { - return restResult(data, FAIL, msg); - } - - public static R fail(int code, String msg) { - return restResult(null, code, msg); - } - - /** - * 返回警告消息 - * - * @param msg 返回内容 - * @return 警告消息 - */ - public static R warn(String msg) { - return restResult(null, 601, msg); - } - - /** - * 返回警告消息 - * - * @param msg 返回内容 - * @param data 数据对象 - * @return 警告消息 - */ - public static R warn(String msg, T data) { - return restResult(data, 601, msg); - } - - private static R restResult(T data, int code, String msg) { - R r = new R<>(); - r.setCode(code); - r.setData(data); - r.setMsg(msg); - return r; - } - - public static Boolean isError(R ret) { - return !isSuccess(ret); - } - - public static Boolean isSuccess(R ret) { - return R.SUCCESS == ret.getCode(); - } -} diff --git a/example/src/main/java/com/inscloudtech/alog/clientdemo/demo/service/impl/DemoUserServiceImpl.java b/example/src/main/java/com/inscloudtech/alog/clientdemo/demo/service/impl/DemoUserServiceImpl.java index 2de8ec1..b96d389 100644 --- a/example/src/main/java/com/inscloudtech/alog/clientdemo/demo/service/impl/DemoUserServiceImpl.java +++ b/example/src/main/java/com/inscloudtech/alog/clientdemo/demo/service/impl/DemoUserServiceImpl.java @@ -4,13 +4,9 @@ import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; import com.inscloudtech.alog.clientdemo.demo.domain.DemoUser; import com.inscloudtech.alog.clientdemo.demo.mapper.DemoUserMapper; import com.inscloudtech.alog.clientdemo.demo.service.DemoUserService; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; @Service -@RequiredArgsConstructor -@Slf4j public class DemoUserServiceImpl extends ServiceImpl implements DemoUserService { } diff --git a/example/src/main/java/com/inscloudtech/alog/clientdemo/utils/BeanCopyUtils.java b/example/src/main/java/com/inscloudtech/alog/clientdemo/utils/BeanCopyUtils.java index 899fd62..64c0dc3 100644 --- a/example/src/main/java/com/inscloudtech/alog/clientdemo/utils/BeanCopyUtils.java +++ b/example/src/main/java/com/inscloudtech/alog/clientdemo/utils/BeanCopyUtils.java @@ -3,9 +3,7 @@ package com.inscloudtech.alog.clientdemo.utils; import com.inscloudtech.alog.common.annotation.NeedRecordField; -import lombok.AccessLevel; -import lombok.NoArgsConstructor; -import lombok.SneakyThrows; + import java.lang.reflect.Field; import java.lang.reflect.Method; @@ -20,7 +18,7 @@ import java.util.*; * * @author inscloudtech */ -@NoArgsConstructor(access = AccessLevel.PRIVATE) + public class BeanCopyUtils { /** @@ -93,30 +91,6 @@ public class BeanCopyUtils { return rObject; } - /** - * 去掉bean中所有属性为字符串的空格 - * @param bean - * @throws Exception - */ - @SneakyThrows - public static void beanAttributeValueTrim(Object bean) { - if(bean!=null){ - //获取所有的字段包括public,private,protected,private - Field[] fields = bean.getClass().getDeclaredFields(); - for (int i = 0; i < fields.length; i++) { - Field f = fields[i]; - if (f.getType().getName().equals("java.lang.String")) { - String key = f.getName();//获取字段名 - Object value = getFieldValue(bean, key); - - if (value == null) - continue; - //全角空格 - setFieldValue(bean, key, value.toString().trim().replaceAll("([　]|\\s|\\u00A0)+","")); - } - } - } - } private static void setFieldValue(Object bean, String fieldName, Object value) diff --git a/example/src/main/java/com/inscloudtech/alog/clientdemo/utils/StringUtils.java b/example/src/main/java/com/inscloudtech/alog/clientdemo/utils/StringUtils.java index 3f1513f..b158d3d 100644 --- a/example/src/main/java/com/inscloudtech/alog/clientdemo/utils/StringUtils.java +++ b/example/src/main/java/com/inscloudtech/alog/clientdemo/utils/StringUtils.java @@ -2,8 +2,7 @@ package com.inscloudtech.alog.clientdemo.utils; -import lombok.AccessLevel; -import lombok.NoArgsConstructor; + import org.springframework.util.AntPathMatcher; import java.util.*; @@ -15,7 +14,7 @@ import java.util.stream.Collectors; * * @author inscloudtech */ -@NoArgsConstructor(access = AccessLevel.PRIVATE) + public class StringUtils { public static final String SEPARATOR = ","; diff --git a/example/src/main/resources/application.yml b/example/src/main/resources/application.yml index 3720117..5ea929e 100644 --- a/example/src/main/resources/application.yml +++ b/example/src/main/resources/application.yml @@ -3,12 +3,11 @@ server: # 数据源配置 spring: datasource: - driver-class-Name: com.mysql.cj.jdbc.Driver + driverClassName: com.mysql.cj.jdbc.Driver url: jdbc:mysql://192.168.3.20:3306/action_log?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8&autoReconnect=true&rewriteBatchedStatements=true username: root password: 123456 # oracle: -# type: ${spring.datasource.type} # driverClassName: oracle.jdbc.OracleDriver # url: jdbc:oracle:thin:@//localhost:1521/XE # username: ROOT @@ -16,13 +15,11 @@ spring: # hikari: # connectionTestQuery: SELECT 1 FROM DUAL # postgres: -# type: ${spring.datasource.type} # driverClassName: org.postgresql.Driver # url: jdbc:postgresql://localhost:5432/postgres?useUnicode=true&characterEncoding=utf8&useSSL=true&autoReconnect=true&reWriteBatchedInserts=true # username: root # password: root # sqlserver: -# type: ${spring.datasource.type} # driverClassName: com.microsoft.sqlserver.jdbc.SQLServerDriver # url: jdbc:sqlserver://localhost:1433;DatabaseName=tempdb;SelectMethod=cursor;encrypt=false;rewriteBatchedStatements=true # username: SA diff --git a/worker/pom.xml b/worker/pom.xml index 46383c5..18d180d 100644 --- a/worker/pom.xml +++ b/worker/pom.xml @@ -54,6 +54,12 @@ spring-boot-configuration-processor true + + + com.baomidou + mybatis-plus-boot-starter + 3.4.3 + diff --git a/worker/src/main/java/com/inscloudtech/alog/worker/store/ActionLogToDbStore.java b/worker/src/main/java/com/inscloudtech/alog/worker/store/ActionLogToDbStore.java new file mode 100644 index 0000000..a821136 --- /dev/null +++ b/worker/src/main/java/com/inscloudtech/alog/worker/store/ActionLogToDbStore.java @@ -0,0 +1,116 @@ +package com.inscloudtech.alog.worker.store; + +import com.inscloudtech.alog.common.utils.AsyncPool; +import com.inscloudtech.alog.common.utils.AsyncWorker; +import com.inscloudtech.alog.common.utils.CollectionUtil; +import com.inscloudtech.alog.worker.db.Db; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import javax.annotation.Resource; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.LongAdder; + +/** + * 解析好的数据暂存和入库 + * @author wuweifeng + * @version 1.0 + * @date 2021-08-21 + */ +@Component +public class ActionLogToDbStore { + /** + * 待入库的数据 + */ + private LinkedBlockingQueue> modelQueue; + /** + * logger + */ + private Logger logger = LoggerFactory.getLogger(getClass()); + /** + * db + */ + @Resource + private Db db; + /** + * 已入库总数量 + */ + private final LongAdder totalInsertCount = new LongAdder(); + /** + * 每批往ck写多少条 + */ + @Value("${clickhouse.batchSize}") + private String batchSize; + /** + * 几个线程去入库 + */ + @Value("${clickhouse.poolSize}") + private String poolSize; + /** + * 间隔几秒入库 + */ + @Value("${clickhouse.insertInterval}") + private int interval; + /** + * 待入库队列长度 + */ + @Value("${queue.preDbSize}") + private int preDbSize; + /** + * 写入队列 + */ + public void offer(Map map) { + boolean success = modelQueue.offer(map); + //如果队列已满，则做其他处理 + if (!success) { + + } + } + + /** + * 入库 + */ + public void beginIntoDb() { + //初始化队列长度 + modelQueue = new LinkedBlockingQueue<>(preDbSize); + int pool = Integer.parseInt(poolSize); + + for (int i = 0; i < pool; i++) { + AsyncPool.asyncDo(() -> { + try { + Thread.sleep(new Random().nextInt(8000)); + } catch (InterruptedException e) { + e.printStackTrace(); + } + while (true) { + try { + List> tempModels = new ArrayList<>(); + //每1s入库一次 + AsyncWorker.drain(modelQueue, tempModels, Integer.parseInt(batchSize), interval, TimeUnit.SECONDS); + if (CollectionUtil.isEmpty(tempModels)) { + continue; + } + + //批量插入 + int successCount = db.insertAll("tracer_model", tempModels); + totalInsertCount.add(successCount); + logger.info("model成功入库 " + tempModels.size() + " 条, 累计已入库 " + totalInsertCount.longValue() + ", 待入库队列size " + modelQueue.size()); + + } catch (Exception e) { + e.printStackTrace(); + } + } + }); + } + + } + + +} diff --git a/worker/src/main/resources/application.yml b/worker/src/main/resources/application.yml index 8c96394..de2575c 100644 --- a/worker/src/main/resources/application.yml +++ b/worker/src/main/resources/application.yml @@ -9,6 +9,13 @@ queue: server: port: 8086 +spring: + datasource: + driverClassName: ru.yandex.clickhouse.ClickHouseDriver + url: jdbc:clickhouse://192.168.3.20:8123/default + username: default + password: + #ck信息，自行修改 clickhouse: url: jdbc:clickhouse://192.168.3.20:8123 diff --git a/worker/src/main/resources/jlog.sql b/worker/src/main/resources/jlog.sql index 171a1c2..7d4ee0d 100644 --- a/worker/src/main/resources/jlog.sql +++ b/worker/src/main/resources/jlog.sql @@ -14,7 +14,7 @@ CREATE TABLE tracer_log ( PARTITION BY toYYYYMMDD ( createTime ) ORDER BY createTime PRIMARY key createTime - TTL createTime + toIntervalDay ( 15 ) + TTL createTime + toIntervalDay ( 15 ); @@ -33,4 +33,34 @@ CREATE TABLE tracer_log ( PARTITION BY toYYYYMMDD ( createTime ) ORDER BY ( uid, createTime ) PRIMARY key uid - TTL createTime + toIntervalDay ( 15 ) + TTL createTime + toIntervalDay ( 15 ); + + +CREATE TABLE action_log ( + log_id Int64 COMMENT '日志主键', + app_id Int64 COMMENT '应用id', + title String COMMENT '分类名称', + method String COMMENT '方法名称', + request_method String COMMENT '请求方式', + operator_type Int8 COMMENT '操作人员（0用户 1系统自动）', + oper_uid String COMMENT '操作人员id', + oper_user_name String COMMENT '操作人员', + dept_name String COMMENT '部门名称', + oper_url String COMMENT '请求URL', + oper_ip String COMMENT '主机地址', + oper_location String COMMENT '操作地点', + oper_param String COMMENT '请求参数', + json_result String COMMENT '返回参数', + status Int8 COMMENT '操作状态（0正常 1异常）', + error_msg String COMMENT '错误消息', + create_time DateTime COMMENT '操作时间', + before_value String COMMENT '更新前数据', + after_value String COMMENT '更新后数据', + business_id String COMMENT '业务主键', + response_time Int8 COMMENT '响应时长' +) + ENGINE = MergeTree() + PARTITION BY toYYYYMMDD ( create_time ) + ORDER BY ( app_id, create_time ) + PRIMARY key log_id; +