suguo
/
jetlinks-community
mirror of https://gitee.com/jetlinks/jetlinks-community.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

refactor: spring-boot3 适配

This commit is contained in:
zhouhao 2025-04-27 17:28:32 +08:00
parent 79e8f22cd5
commit 587c4f4d48
382 changed files with 12212 additions and 7226 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -29,4 +29,5 @@ docker/data
!demo-protocol-1.0.jar
application-local.yml
dev/
.DS_Store
.DS_Store
.java-version

18
.run/JetLinksApplication.run.xml Normal file
View File

@ -0,0 +1,18 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="JetLinksApplication" type="SpringBootApplicationConfigurationType" factoryName="Spring Boot">
<option name="ACTIVE_PROFILES" value="dev,local" />
<additionalParameters>
<param>
<option name="enabled" value="true" />
<option name="name" value="server.port" />
<option name="value" value="8801" />
</param>
</additionalParameters>
<module name="jetlinks-standalone" />
<option name="SPRING_BOOT_MAIN_CLASS" value="org.jetlinks.community.standalone.JetLinksApplication" />
<option name="VM_PARAMETERS" value="--add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/java.util.concurrent=ALL-UNNAMED --add-opens java.base/java.io=ALL-UNNAMED --add-opens java.base/java.net=ALL-UNNAMED --add-opens java.base/java.text=ALL-UNNAMED --add-opens java.base/java.math=ALL-UNNAMED --add-opens java.scripting/javax.script=ALL-UNNAMED -XX:+EnableDynamicAgentLoading" />
<method v="2">
<option name="Make" enabled="true" />
</method>
</configuration>
</component>

6
README.md
View File

@ -14,10 +14,10 @@
[![QQ②群324606263](https://img.shields.io/badge/QQ②群-324606263-brightgreen)](https://qm.qq.com/cgi-bin/qm/qr?k=IMas2cH-TNsYxUcY8lRbsXqPnA2sGHYQ&jump_from=webapi)
[![QQ①群2021514](https://img.shields.io/badge/QQ①群-2021514-brightgreen)](https://qm.qq.com/cgi-bin/qm/qr?k=LGf0OPQqvLGdJIZST3VTcypdVWhdfAOG&jump_from=webapi)
JetLinks 基于Java8,Spring Boot 2.x,WebFlux,Netty,Vert.x,Reactor等开发,
JetLinks 基于Java 17,Spring Boot 3.x,WebFlux,Netty,Vert.x,Reactor等开发,
是一个开箱即用,可二次开发的企业级物联网基础平台。平台实现了物联网相关的众多基础功能,
能帮助你快速建立物联网相关业务系统。
## 核心特性
@ -36,7 +36,7 @@ TCP/UDP/MQTT/HTTP、TLS/DTLS、不同厂商、不同设备、不同报文、统
## 技术栈
1. [Spring Boot 2.7.x](https://spring.io/projects/spring-boot)
1. [Spring Boot 3.4.x](https://spring.io/projects/spring-boot)
2. [Spring WebFlux](https://spring.io/) 响应式Web支持
3. [R2DBC](https://r2dbc.io/) 响应式关系型数据库驱动
4. [Project Reactor](https://projectreactor.io/) 响应式编程框架

4
docker/run-all/docker-compose.yml
View File

@ -48,7 +48,7 @@ services:
POSTGRES_DB: jetlinks
TZ: Asia/Shanghai
ui:
image: registry.cn-shenzhen.aliyuncs.com/jetlinks/jetlinks-ui-vue:2.3.0-SNAPSHOT
image: registry.cn-shenzhen.aliyuncs.com/jetlinks/jetlinks-ui-vue:2.10.0-SNAPSHOT
container_name: jetlinks-ce-ui
ports:
- 9000:80
@ -59,7 +59,7 @@ services:
links:
- jetlinks:jetlinks
jetlinks:
image: registry.cn-shenzhen.aliyuncs.com/jetlinks/jetlinks-community:2.3.0-SNAPSHOT
image: registry.cn-shenzhen.aliyuncs.com/jetlinks/jetlinks-community:2.10.0-SNAPSHOT
container_name: jetlinks-ce
ports:

13
jetlinks-components/common-component/pom.xml
View File

@ -5,7 +5,7 @@
<parent>
<artifactId>jetlinks-components</artifactId>
<groupId>org.jetlinks.community</groupId>
<version>2.3.0-SNAPSHOT</version>
<version>2.10.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
@ -80,9 +80,12 @@
<dependency>
<groupId>com.cronutils</groupId>
<artifactId>cron-utils</artifactId>
<version>9.2.0</version>
<scope>compile</scope>
<exclusions>
<exclusion>
<groupId>org.glassfish</groupId>
<artifactId>jakarta.el</artifactId>
</exclusion>
<exclusion>
<groupId>org.glassfish</groupId>
<artifactId>javax.el</artifactId>
@ -95,5 +98,11 @@
<artifactId>spring-data-redis</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.glassfish.expressly</groupId>
<artifactId>expressly</artifactId>
</dependency>
</dependencies>
</project>

9
jetlinks-components/common-component/src/main/java/org/jetlinks/community/PropertyConstants.java
View File

@ -38,13 +38,6 @@ public interface PropertyConstants {
*/
Key<List<Map<String, Object>>> relations = Key.of("relations");
/**
* 租户ID
*
* @see org.jetlinks.pro.tenant.TenantMember
*/
Key<List<String>> tenantId = Key.of("tenantId");
//分组ID
Key<List<String>> groupId = Key.of("groupId");
@ -71,7 +64,7 @@ public interface PropertyConstants {
/**
* 设备接入方式
*
* @see org.jetlinks.pro.gateway.supports.DeviceGatewayProvider#getId
* @see org.jetlinks.community.gateway.supports.DeviceGatewayProvider#getId
*/
Key<String> accessProvider = Key.of("accessProvider");

2
jetlinks-components/common-component/src/main/java/org/jetlinks/community/PropertyMetric.java
View File

@ -6,7 +6,7 @@ import org.hswebframework.web.bean.FastBeanCopier;
import org.jetlinks.community.utils.ConverterUtils;
import org.springframework.util.StringUtils;
import javax.validation.constraints.NotBlank;
import jakarta.validation.constraints.NotBlank;
import java.util.Map;
import java.util.function.Function;

4
jetlinks-components/common-component/src/main/java/org/jetlinks/community/TimerSpec.java
View File

@ -28,8 +28,8 @@ import reactor.core.scheduler.Scheduler;
import reactor.core.scheduler.Schedulers;
import javax.annotation.Nonnull;
import javax.validation.constraints.NotBlank;
import javax.validation.constraints.NotNull;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.NotNull;
import java.io.Serializable;
import java.time.*;
import java.time.temporal.ChronoUnit;

29
jetlinks-components/common-component/src/main/java/org/jetlinks/community/configuration/CommonConfiguration.java
View File

@ -13,6 +13,8 @@ import org.hswebframework.web.dict.EnumDict;
import org.hswebframework.web.dict.defaults.DefaultItemDefine;
import org.jetlinks.community.Interval;
import org.jetlinks.community.JvmErrorException;
import org.jetlinks.community.command.CommandSupportManagerProvider;
import org.jetlinks.community.command.CommandSupportManagerProviders;
import org.jetlinks.community.command.register.CommandServiceEndpointRegister;
import org.jetlinks.community.config.ConfigManager;
import org.jetlinks.community.config.ConfigScopeCustomizer;
@ -20,8 +22,11 @@ import org.jetlinks.community.config.ConfigScopeProperties;
import org.jetlinks.community.config.SimpleConfigManager;
import org.jetlinks.community.config.entity.ConfigEntity;
import org.jetlinks.community.dictionary.DictionaryJsonDeserializer;
import org.jetlinks.community.form.type.FieldTypeProvider;
import org.jetlinks.community.reactorql.aggregation.InternalAggregationSupports;
import org.jetlinks.community.reactorql.function.InternalFunctionSupport;
import org.jetlinks.community.reactorql.term.TermTypeSupport;
import org.jetlinks.community.reactorql.term.TermTypes;
import org.jetlinks.community.reference.DataReferenceManager;
import org.jetlinks.community.reference.DataReferenceProvider;
import org.jetlinks.community.reference.DefaultDataReferenceManager;
@ -45,6 +50,7 @@ import org.springframework.boot.autoconfigure.AutoConfiguration;
import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
import org.springframework.boot.autoconfigure.jackson.Jackson2ObjectMapperBuilderCustomizer;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.ApplicationContextAware;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.redis.core.ReactiveRedisOperations;
@ -185,15 +191,20 @@ public class CommonConfiguration {
}
@Bean
public BeanPostProcessor globalReactorQlFeatureRegister() {
return new BeanPostProcessor() {
@Override
public Object postProcessAfterInitialization(@Nonnull Object bean, @Nonnull String beanName) throws BeansException {
if (bean instanceof Feature) {
DefaultReactorQLMetadata.addGlobal(((Feature) bean));
}
return bean;
}
public ApplicationContextAware staticBeanRegister() {
return ctx -> {
ctx.getBeanProvider(Feature.class)
.forEach(DefaultReactorQLMetadata::addGlobal);
ctx.getBeanProvider(CommandSupportManagerProvider.class)
.forEach(CommandSupportManagerProviders::register);
ctx.getBeanProvider(TermTypeSupport.class)
.forEach(TermTypes::register);
ctx.getBeanProvider(FieldTypeProvider.class)
.forEach(provider -> FieldTypeProvider.supports.register(provider.getProvider(), provider));
};
}

15
jetlinks-components/common-component/src/main/java/org/jetlinks/community/dictionary/DictionaryConfiguration.java
View File

@ -1,5 +1,8 @@
package org.jetlinks.community.dictionary;
import org.hswebframework.web.crud.events.EntityEventListenerCustomizer;
import org.hswebframework.web.dictionary.entity.DictionaryEntity;
import org.hswebframework.web.dictionary.entity.DictionaryItemEntity;
import org.hswebframework.web.dictionary.service.DefaultDictionaryItemService;
import org.hswebframework.web.dictionary.service.DefaultDictionaryService;
import org.springframework.beans.factory.ObjectProvider;
@ -9,15 +12,22 @@ import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
@AutoConfiguration
public class DictionaryConfiguration {
@Configuration
@AutoConfiguration
@ConditionalOnClass(DefaultDictionaryItemService.class)
//@ConditionalOnBean(DefaultDictionaryItemService.class)
public static class DictionaryManagerConfiguration {
@Bean
public EntityEventListenerCustomizer dictionaryEntityEventListenerCustomizer() {
return configure -> {
configure.enable(DictionaryItemEntity.class);
configure.enable(DictionaryEntity.class);
};
}
@Bean
public DictionaryEventHandler dictionaryEventHandler(DefaultDictionaryItemService service) {
@ -44,5 +54,6 @@ public class DictionaryConfiguration {
DefaultDictionaryItemService itemService) {
return new DictionaryInitManager(initInfo, defaultDictionaryService, itemService);
}
}
}

32
jetlinks-components/common-component/src/main/java/org/jetlinks/community/dictionary/DictionaryEventHandler.java
View File

@ -7,16 +7,21 @@ import org.hswebframework.web.crud.events.*;
import org.hswebframework.web.dictionary.entity.DictionaryEntity;
import org.hswebframework.web.dictionary.entity.DictionaryItemEntity;
import org.hswebframework.web.dictionary.service.DefaultDictionaryItemService;
import org.hswebframework.web.dictionary.service.DefaultDictionaryService;
import org.hswebframework.web.exception.BusinessException;
import org.springframework.context.event.EventListener;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import java.util.Collections;
import java.util.List;
/**
* @author bestfeng
*/
@AllArgsConstructor
public class DictionaryEventHandler implements EntityEventListenerCustomizer {
public class DictionaryEventHandler {
private final DefaultDictionaryItemService itemService;
@ -68,28 +73,23 @@ public class DictionaryEventHandler implements EntityEventListenerCustomizer {
);
}
@Override
public void customize(EntityEventListenerConfigure configure) {
configure.enable(DictionaryItemEntity.class);
configure.enable(DictionaryEntity.class);
}
/**
* 监听字典删除前事件，阻止删除分类标识为系统的字典
*
* @param event 字典删除前事件
*/
@EventListener
public void handleDictionaryBeforeDelete(EntityBeforeDeleteEvent<DictionaryEntity> event) {
event.async(
Flux.fromIterable(event.getEntity())
.any(dictionary ->
StringUtils.equals(dictionary.getClassified(), DictionaryConstants.CLASSIFIED_SYSTEM))
.flatMap(any -> {
if (any) {
return Mono.error(() -> new BusinessException("error.system_dictionary_can_not_delete"));
}
return Mono.empty();
})
Flux.fromIterable(event.getEntity())
.any(dictionary ->
StringUtils.equals(dictionary.getClassified(), DictionaryConstants.CLASSIFIED_SYSTEM))
.flatMap(any -> {
if (any) {
return Mono.error(() -> new BusinessException("error.system_dictionary_can_not_delete"));
}
return Mono.empty();
})
);
}
}

26
jetlinks-components/common-component/src/main/java/org/jetlinks/community/reactorql/EnableReactorQL.java
View File

@ -1,26 +0,0 @@
package org.jetlinks.community.reactorql;
import org.springframework.context.annotation.Import;
import java.lang.annotation.*;
/**
* 在配置类上加上此注解,并指定{@link EnableReactorQL#value()},将扫描指定包下注解了{@link ReactorQLOperation}的接口类,
* 并生成代理对象注入到spring中.
*
* @author zhouhao
* @since 1.6
* @see ReactorQL
* @see ReactorQLOperation
*/
@Target({ElementType.TYPE})
@Retention(RetentionPolicy.RUNTIME)
@Documented
@Inherited
@Import(ReactorQLBeanDefinitionRegistrar.class)
public @interface EnableReactorQL {
/**
* @return 扫描的包名
*/
String[] value() default {};
}

37
jetlinks-components/common-component/src/main/java/org/jetlinks/community/reactorql/ReactorQL.java
View File

@ -1,37 +0,0 @@
package org.jetlinks.community.reactorql;
import java.lang.annotation.*;
/**
* 在接口的方法上注解,使用sql语句来处理参数
*
* @author zhouhao
* @see org.jetlinks.reactor.ql.ReactorQL
* @see ReactorQLOperation
* @since 1.6
*/
@Target({ElementType.METHOD, ElementType.TYPE})
@Retention(RetentionPolicy.RUNTIME)
@Documented
@Inherited
public @interface ReactorQL {
/**
* 使用SQL语句来处理{@link reactor.core.publisher.Flux}操作.例如分组聚合.
* <a href="https://doc.jetlinks.cn/dev-guide/reactor-ql.html">查看文档说明</a>
*
* <pre>
* select count(1) total,name from "arg0" group by name
* </pre>
* <p>
* <p>
* 当方法有参数时,可通过arg{index}来获取参数,如:
* <pre>
* select name newName from "arg0" where id = :arg1
* </pre>
*
* @return SQL语句
*/
String[] value();
}

59
jetlinks-components/common-component/src/main/java/org/jetlinks/community/reactorql/ReactorQLBeanDefinitionRegistrar.java
View File

@ -1,59 +0,0 @@
package org.jetlinks.community.reactorql;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.support.AbstractBeanDefinition;
import org.springframework.beans.factory.support.BeanDefinitionRegistry;
import org.springframework.beans.factory.support.RootBeanDefinition;
import org.springframework.context.annotation.ImportBeanDefinitionRegistrar;
import org.springframework.context.index.CandidateComponentsIndex;
import org.springframework.context.index.CandidateComponentsIndexLoader;
import org.springframework.core.type.AnnotationMetadata;
import javax.annotation.Nonnull;
import java.util.Arrays;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
@Slf4j
public class ReactorQLBeanDefinitionRegistrar implements ImportBeanDefinitionRegistrar {
@Override
@SneakyThrows
public void registerBeanDefinitions(AnnotationMetadata importingClassMetadata, @Nonnull BeanDefinitionRegistry registry) {
Map<String, Object> attr = importingClassMetadata.getAnnotationAttributes(EnableReactorQL.class.getName());
if (attr == null) {
return;
}
String[] packages = (String[]) attr.get("value");
CandidateComponentsIndex index = CandidateComponentsIndexLoader.loadIndex(org.springframework.util.ClassUtils.getDefaultClassLoader());
if (null == index) {
return;
}
Set<String> path = Arrays.stream(packages)
.flatMap(str -> index
.getCandidateTypes(str, ReactorQLOperation.class.getName())
.stream())
.collect(Collectors.toSet());
for (String className : path) {
Class<?> type = org.springframework.util.ClassUtils.forName(className, null);
if (!type.isInterface() || type.getAnnotation(ReactorQLOperation.class) == null) {
continue;
}
RootBeanDefinition definition = new RootBeanDefinition();
definition.setTargetType(type);
definition.setBeanClass(ReactorQLFactoryBean.class);
definition.setAutowireMode(AbstractBeanDefinition.AUTOWIRE_BY_TYPE);
definition.getPropertyValues().add("target", type);
if (!registry.containsBeanDefinition(type.getName())) {
log.debug("register ReactorQL Operator {}", type);
registry.registerBeanDefinition(type.getName(), definition);
}
}
}
}

164
jetlinks-components/common-component/src/main/java/org/jetlinks/community/reactorql/ReactorQLFactoryBean.java
View File

@ -1,164 +0,0 @@
package org.jetlinks.community.reactorql;
import lombok.Getter;
import lombok.Setter;
import lombok.SneakyThrows;
import org.hswebframework.web.bean.FastBeanCopier;
import org.jetlinks.reactor.ql.ReactorQLContext;
import org.reactivestreams.Publisher;
import org.springframework.beans.factory.FactoryBean;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.core.LocalVariableTableParameterNameDiscoverer;
import org.springframework.core.Ordered;
import org.springframework.core.ParameterNameDiscoverer;
import org.springframework.core.ResolvableType;
import org.springframework.util.ClassUtils;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import java.lang.invoke.MethodHandle;
import java.lang.invoke.MethodHandles;
import java.lang.reflect.Constructor;
import java.lang.reflect.Method;
import java.lang.reflect.Proxy;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Function;
public class ReactorQLFactoryBean implements FactoryBean<Object>, InitializingBean, Ordered {
@Getter
@Setter
private Class<?> target;
private Object proxy;
private static final ParameterNameDiscoverer nameDiscoverer = new LocalVariableTableParameterNameDiscoverer();
public ReactorQLFactoryBean() {
}
@Override
public Object getObject() {
return proxy;
}
@Override
public Class<?> getObjectType() {
return target;
}
@Override
public void afterPropertiesSet() {
Map<Method, Function<Object[], Object>> cache = new ConcurrentHashMap<>();
this.proxy = Proxy
.newProxyInstance(ClassUtils.getDefaultClassLoader(),
new Class[]{target},
(proxy, method, args) ->
cache
.computeIfAbsent(method, mtd -> createInvoker(target, mtd, mtd.getAnnotation(ReactorQL.class)))
.apply(args));
}
@SneakyThrows
private Function<Object[], Object> createInvoker(Class<?> type, Method method, ReactorQL ql) {
if (method.isDefault() || ql == null) {
Constructor<MethodHandles.Lookup> constructor = MethodHandles.Lookup.class
.getDeclaredConstructor(Class.class);
constructor.setAccessible(true);
MethodHandles.Lookup lookup = constructor.newInstance(type);
MethodHandle handle = lookup
.in(type)
.unreflectSpecial(method, type)
.bindTo(proxy);
return args -> {
try {
return handle.invokeWithArguments(args);
} catch (Throwable e) {
return Mono.error(e);
}
};
}
ResolvableType returnType = ResolvableType.forMethodReturnType(method);
if (returnType.toClass() != Mono.class && returnType.toClass() != Flux.class) {
throw new UnsupportedOperationException("方法返回值必须为Mono或者Flux");
}
Class<?> genericType = returnType.getGeneric(0).toClass();
Function<Map<String, Object>, ?> mapper;
if (genericType == Map.class || genericType == Object.class) {
mapper = Function.identity();
} else {
mapper = map -> FastBeanCopier.copy(map, genericType);
}
Function<Flux<?>, Publisher<?>> resultMapper =
returnType.resolve() == Mono.class
? flux -> flux.take(1).singleOrEmpty()
: flux -> flux;
String[] names = nameDiscoverer.getParameterNames(method);
try {
org.jetlinks.reactor.ql.ReactorQL reactorQL =
org.jetlinks.reactor.ql.ReactorQL
.builder()
.sql(ql.value())
.build();
return args -> {
Map<String, Object> argsMap = new HashMap<>();
ReactorQLContext context = ReactorQLContext.ofDatasource(name -> {
if (args.length == 0) {
return Flux.just(1);
}
if (args.length == 1) {
return convertToFlux(args[0]);
}
return convertToFlux(argsMap.get(name));
});
for (int i = 0; i < args.length; i++) {
String indexName = "arg" + i;
String name = names == null ? indexName : names[i];
context.bind(i, args[i]);
context.bind(name, args[i]);
context.bind(indexName, args[i]);
argsMap.put(names == null ? indexName : names[i], args[i]);
argsMap.put(indexName, args[i]);
}
return reactorQL.start(context)
.map(record -> mapper.apply(record.asMap()))
.as(resultMapper);
};
} catch (Throwable e) {
throw new IllegalArgumentException(
"create ReactorQL method [" + method + "] error,sql:\n" + (String.join(" ", ql.value())), e);
}
}
protected Flux<Object> convertToFlux(Object arg) {
if (arg == null) {
return Flux.empty();
}
if (arg instanceof Publisher) {
return Flux.from((Publisher<?>) arg);
}
if (arg instanceof Iterable) {
return Flux.fromIterable(((Iterable<?>) arg));
}
if (arg instanceof Object[]) {
return Flux.fromArray(((Object[]) arg));
}
return Flux.just(arg);
}
@Override
public int getOrder() {
return Ordered.LOWEST_PRECEDENCE;
}
}

22
jetlinks-components/common-component/src/main/java/org/jetlinks/community/reactorql/ReactorQLOperation.java
View File

@ -1,22 +0,0 @@
package org.jetlinks.community.reactorql;
import org.springframework.stereotype.Indexed;
import java.lang.annotation.*;
/**
* 在接口上添加此注解,开启使用sql来处理reactor数据
*
* @author zhouhao
* @see ReactorQL
* @see EnableReactorQL
* @since 1.6
*/
@Target({ElementType.TYPE})
@Retention(RetentionPolicy.RUNTIME)
@Documented
@Inherited
@Indexed
public @interface ReactorQLOperation {
}

6
jetlinks-components/common-component/src/main/java/org/jetlinks/community/reactorql/term/TermValue.java
View File

@ -83,7 +83,7 @@ public class TermValue implements Serializable {
/**
* 和manual一样,
* 兼容{@link org.jetlinks.pro.relation.utils.VariableSource.Source#fixed}
* 兼容{@link org.jetlinks.community.relation.utils.VariableSource.Source#fixed}
*/
fixed,
manual,
@ -91,14 +91,14 @@ public class TermValue implements Serializable {
metric,
variable,
/**
* 和variable一样,兼容{@link org.jetlinks.pro.relation.utils.VariableSource.Source#upper}
* 和variable一样,兼容{@link org.jetlinks.community.relation.utils.VariableSource.Source#upper}
*/
upper,
/**
* 函数
*
* @see org.jetlinks.pro.reactorql.function.FunctionSupport
* @see org.jetlinks.community.reactorql.function.FunctionSupport
*/
function
}

3
jetlinks-components/common-component/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports
View File

@ -1 +1,2 @@
org.jetlinks.community.configuration.CommonConfiguration
org.jetlinks.community.configuration.CommonConfiguration
org.jetlinks.community.dictionary.DictionaryConfiguration

4
jetlinks-components/configure-component/pom.xml
View File

@ -5,7 +5,7 @@
<parent>
<artifactId>jetlinks-components</artifactId>
<groupId>org.jetlinks.community</groupId>
<version>2.3.0-SNAPSHOT</version>
<version>2.10.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
@ -13,8 +13,6 @@
<artifactId>configure-component</artifactId>
<properties>
<maven.compiler.source>8</maven.compiler.source>
<maven.compiler.target>8</maven.compiler.target>
</properties>
<dependencies>

5
jetlinks-components/configure-component/src/main/java/org/jetlinks/community/configure/JetLinksCommonConfiguration.java
View File

@ -1,5 +1,6 @@
package org.jetlinks.community.configure;
import org.jetlinks.community.configure.compatible.PathCompatibleFilter;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import reactor.core.scheduler.Scheduler;
@ -13,4 +14,8 @@ public class JetLinksCommonConfiguration {
return Schedulers.parallel();
}
@Bean
public PathCompatibleFilter pathCompatibleFilter(){
return new PathCompatibleFilter();
}
}

34
jetlinks-components/configure-component/src/main/java/org/jetlinks/community/configure/compatible/PathCompatibleFilter.java Normal file
View File

@ -0,0 +1,34 @@
package org.jetlinks.community.configure.compatible;
import org.springframework.web.server.ServerWebExchange;
import org.springframework.web.server.WebFilter;
import org.springframework.web.server.WebFilterChain;
import reactor.core.publisher.Mono;
import javax.annotation.Nonnull;
/**
* 兼容处理路径结尾为/的请求
*
* @since 2.10
*/
public class PathCompatibleFilter implements WebFilter {
@Override
@Nonnull
public Mono<Void> filter(ServerWebExchange exchange, @Nonnull WebFilterChain chain) {
String path = exchange.getRequest().getPath().toString();
if (!path.equals("/") && path.endsWith("/")) {
path = path.substring(0, path.length() - 1);
return chain.filter(
exchange
.mutate()
.request(exchange
.getRequest()
.mutate()
.path(path)
.build())
.build());
}
return chain.filter(exchange);
}
}

36
jetlinks-components/configure-component/src/main/java/org/jetlinks/community/configure/device/DeviceClusterConfiguration.java
View File

@ -13,20 +13,22 @@ import org.jetlinks.core.device.session.DeviceSessionManager;
import org.jetlinks.core.message.interceptor.DeviceMessageSenderInterceptor;
import org.jetlinks.core.rpc.RpcManager;
import org.jetlinks.core.server.MessageHandler;
import org.jetlinks.supports.cluster.ClusterDeviceOperationBroker;
import org.jetlinks.core.things.ThingRpcSupportChain;
import org.jetlinks.supports.cluster.ClusterDeviceRegistry;
import org.jetlinks.supports.cluster.RpcDeviceOperationBroker;
import org.jetlinks.supports.scalecube.ExtendedCluster;
import org.jetlinks.supports.server.ClusterSendToDeviceMessageHandler;
import org.jetlinks.supports.server.DecodedClientMessageHandler;
import org.springframework.beans.BeansException;
import org.springframework.beans.factory.SmartInitializingSingleton;
import org.springframework.beans.factory.config.BeanPostProcessor;
import org.springframework.boot.autoconfigure.AutoConfiguration;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration(proxyBeanMethods = false)
@AutoConfiguration
@ConditionalOnBean(ProtocolSupports.class)
public class DeviceClusterConfiguration {
@ -46,18 +48,22 @@ public class DeviceClusterConfiguration {
@Bean
@ConditionalOnBean(ClusterDeviceRegistry.class)
public BeanPostProcessor interceptorRegister(ClusterDeviceRegistry registry) {
return new BeanPostProcessor() {
@Override
public Object postProcessAfterInitialization(Object bean, String beanName) throws BeansException {
if (bean instanceof DeviceMessageSenderInterceptor) {
registry.addInterceptor(((DeviceMessageSenderInterceptor) bean));
}
if (bean instanceof DeviceStateChecker) {
registry.addStateChecker(((DeviceStateChecker) bean));
}
return bean;
}
public SmartInitializingSingleton interceptorRegister(ApplicationContext context,
ClusterDeviceRegistry registry) {
return ()->{
context.getBeansOfType(DeviceMessageSenderInterceptor.class)
.values()
.forEach(registry::addInterceptor);
context.getBeansOfType(DeviceStateChecker.class)
.values()
.forEach(registry::addStateChecker);
context.getBeansOfType(ThingRpcSupportChain.class)
.values()
.forEach(registry::addRpcChain);
};
}

10
jetlinks-components/configure-component/src/main/java/org/jetlinks/community/configure/trace/TraceWebFilter.java
View File

@ -15,7 +15,7 @@ public class TraceWebFilter implements WebFilter, Ordered {
public Mono<Void> filter(ServerWebExchange exchange,
WebFilterChain chain) {
// /http/method/path
String spanName = "/http/"+exchange.getRequest().getMethodValue() + exchange.getRequest().getPath().value();
String spanName = "/http/"+exchange.getRequest().getMethod().name() + exchange.getRequest().getPath().value();
ServerHttpRequest.Builder requestCopy = exchange
.getRequest()
@ -31,8 +31,12 @@ public class TraceWebFilter implements WebFilter, Ordered {
//创建跟踪信息
.as(MonoTracer.create(spanName))
//从请求头中追加上级跟踪信息
.as(MonoTracer.createWith(exchange.getRequest().getHeaders(),HttpHeadersGetter.INSTANCE));
.contextWrite(ctx -> {
return TraceHolder.readToContext(
ctx,
exchange.getRequest().getHeaders(),
HttpHeadersGetter.INSTANCE);
});
}
@Override

2
jetlinks-components/dashboard-component/pom.xml
View File

@ -5,7 +5,7 @@
<parent>
<artifactId>jetlinks-components</artifactId>
<groupId>org.jetlinks.community</groupId>
<version>2.3.0-SNAPSHOT</version>
<version>2.10.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>

74
jetlinks-components/datasource-component/pom.xml Normal file
View File

@ -0,0 +1,74 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>jetlinks-components</artifactId>
<groupId>org.jetlinks.community</groupId>
<version>2.10.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>datasource-component</artifactId>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-mongodb-reactive</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.jetlinks.community</groupId>
<artifactId>common-component</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.zaxxer</groupId>
<artifactId>HikariCP</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.r2dbc</groupId>
<artifactId>r2dbc-spi</artifactId>
</dependency>
<dependency>
<groupId>io.r2dbc</groupId>
<artifactId>r2dbc-pool</artifactId>
</dependency>
<dependency>
<groupId>io.r2dbc</groupId>
<artifactId>r2dbc-h2</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.data</groupId>
<artifactId>spring-data-r2dbc</artifactId>
</dependency>
<dependency>
<groupId>org.jetlinks.community</groupId>
<artifactId>rule-engine-component</artifactId>
<version>${project.version}</version>
<optional>true</optional>
</dependency>
</dependencies>
</project>

92
jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/AbstractDataSource.java Normal file
View File

@ -0,0 +1,92 @@
package org.jetlinks.community.datasource;
import lombok.Generated;
import lombok.Getter;
import org.hswebframework.web.bean.FastBeanCopier;
import org.jetlinks.core.command.AbstractCommandSupport;
import reactor.core.publisher.Mono;
import javax.annotation.Nonnull;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@Getter
public abstract class AbstractDataSource<C> extends AbstractCommandSupport implements DataSource {
private final String id;
private C config;
private volatile boolean disposed;
public AbstractDataSource(String id,
C config) {
this.id = id;
this.config = config;
}
@Override
@Generated
public final String getId() {
return id;
}
@Override
public abstract DataSourceType getType();
@Override
public final void dispose() {
disposed = true;
doOnDispose();
}
@Override
public final boolean isDisposed() {
return disposed;
}
@Generated
public final C getConfig() {
return config;
}
@SuppressWarnings("all")
public C copyConfig() {
return (C) FastBeanCopier.copy(config, config.getClass());
}
public final void setConfig(C config) {
C old = this.config;
this.config = config;
handleSetConfig(old, config);
}
@Override
public final Mono<DataSourceState> state() {
if (isDisposed()) {
return Mono.just(DataSourceState.stopped);
}
return this.checkState();
}
protected Mono<DataSourceState> checkState() {
return Mono.just(DataSourceState.ok);
}
protected void handleSetConfig(C oldConfig, C newConfig) {
}
protected void doOnDispose() {
}
@Override
protected <R> R executeUndefinedCommand(@Nonnull org.jetlinks.core.command.Command<R> command) {
return super.executeUndefinedCommand(command);
}
}

75
jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DataSource.java Normal file
View File

@ -0,0 +1,75 @@
package org.jetlinks.community.datasource;
import org.jetlinks.core.command.CommandException;
import org.jetlinks.core.command.CommandSupport;
import reactor.core.Disposable;
import reactor.core.publisher.Mono;
import javax.annotation.Nonnull;
/**
* 统一数据源接口定义,{@link DataSource#getType()}标识数据源类型.
* <p>
* 数据源统一管理,请勿手动调用{@link DataSource#dispose()}
*
* @author zhouhao
* @since 1.10
*/
public interface DataSource extends CommandSupport, Disposable {
/**
* @return 数据源ID
*/
String getId();
/**
* @return 数据源类型
*/
DataSourceType getType();
/**
* 执行指令,具体指令有对应的数据源实现定义.
*
* @param command 指令
* @param <R> 结果类型
* @return void
* @see UnsupportedOperationException
*/
@Nonnull
@Override
default <R> R execute(@Nonnull org.jetlinks.core.command.Command<R> command) {
throw new CommandException.NoStackTrace(this, command, "error.unsupported_command");
}
/**
* 获取数据源状态
*
* @return 状态
* @see DataSourceState
*/
default Mono<DataSourceState> state() {
return Mono.just(DataSourceState.ok);
}
/**
* 判断数据源是为指定的类型
*
* @param target 类型
* @return 是否为指定的类型
*/
default boolean isWrapperFor(java.lang.Class<?> target) {
return target.isInstance(this);
}
/**
* 按指定类型拆箱数据源,返回对应的数据源。如果类型不一致,可能抛出{@link ClassCastException}
*
* @param target 目标类型
* @param <T> T
* @return 数据源
*/
default <T> T unwrap(Class<T> target) {
return target.cast(this);
}
}

22
jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DataSourceConfig.java Normal file
View File

@ -0,0 +1,22 @@
package org.jetlinks.community.datasource;
import lombok.Generated;
import lombok.Getter;
import lombok.Setter;
import org.jetlinks.community.ValueObject;
import java.util.Map;
@Getter
@Setter
@Generated
public class DataSourceConfig implements ValueObject {
private String id;
private String typeId;
private Map<String,Object> configuration;
@Override
public Map<String, Object> values() {
return configuration;
}
}

36
jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DataSourceConfigManager.java Normal file
View File

@ -0,0 +1,36 @@
package org.jetlinks.community.datasource;
import reactor.core.Disposable;
import reactor.core.publisher.Mono;
import java.util.function.BiFunction;
/**
* 数据源配置管理器,统一管理数据源配置
*
* @author zhouhao
* @since 1.10
*/
public interface DataSourceConfigManager {
/**
* 根据类型ID和数据源ID获取配置
*
* @param typeId 类型ID
* @param datasourceId 数据源ID
* @return 配置信息
*/
Mono<DataSourceConfig> getConfig(String typeId, String datasourceId);
/**
* 监听配置变化，当有配置变化后将调用回调参数
*
* @param callback 回调参数
*/
Disposable doOnConfigChanged(BiFunction<ConfigState, DataSourceConfig,Mono<Void>> callback);
enum ConfigState{
normal,
disabled
}
}

46
jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DataSourceConstants.java Normal file
View File

@ -0,0 +1,46 @@
package org.jetlinks.community.datasource;
import org.jetlinks.core.command.Command;
import org.jetlinks.core.command.CommandSupport;
import org.jetlinks.core.command.CommandUtils;
import org.jetlinks.core.metadata.FunctionMetadata;
import org.jetlinks.core.metadata.SimpleFunctionMetadata;
import org.jetlinks.community.command.CommandSupportManagerProviders;
import reactor.core.publisher.Mono;
import java.util.function.Consumer;
public interface DataSourceConstants {
interface Commands {
static String createCommandProvider(String dataSourceId) {
return "datasource$" + dataSourceId;
}
static Mono<CommandSupport> getCommandSupport(String datasourceId) {
return CommandSupportManagerProviders
.getCommandSupport(createCommandProvider(datasourceId));
}
static Mono<CommandSupport> getCommandSupport(String datasourceId, String supportId) {
return CommandSupportManagerProviders
.getCommandSupport(createCommandProvider(datasourceId), supportId);
}
}
interface Metadata {
static FunctionMetadata create(@SuppressWarnings("all") Class<? extends Command> cmdType,
Consumer<SimpleFunctionMetadata> handler) {
SimpleFunctionMetadata metadata = new SimpleFunctionMetadata();
metadata.setId(CommandUtils.getCommandIdByType(cmdType));
metadata.setName(metadata.getId());
handler.accept(metadata);
return metadata;
}
}
}

84
jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DataSourceManager.java Normal file
View File

@ -0,0 +1,84 @@
package org.jetlinks.community.datasource;
import reactor.core.publisher.Flux;
import org.jetlinks.community.datasource.exception.DataSourceNotExistException;
import reactor.core.publisher.Mono;
import java.util.List;
/**
* 数据源管理器,用于统一管理数据源
*
* @author zhouhao
* @since 1.10
*/
public interface DataSourceManager {
/**
* 获取支持的数据源类型
*
* @return 数据源类型
*/
List<DataSourceType> getSupportedType();
/**
* 根据类型获取数据眼供应商
*
* @param typeId 类型ID
* @return 数据源供应商
*/
DataSourceProvider getProvider(String typeId);
/**
* 根据类型ID获取已存在的数据源
*
* @param typeId 类型ID
* @return 数据源列表
*/
Flux<DataSource> getDataSources(String typeId);
/**
* 获取指定的数据源,如果数据源不存在则返回{@link Mono#empty()}
*
* @param type 数据源类型
* @param datasourceId 数据源ID
* @return 数据源
*/
Mono<DataSource> getDataSource(DataSourceType type, String datasourceId);
/**
* 获取指定的数据源,如果数据源不存在则抛出异常{@link DataSourceNotExistException}
*
* @param type 数据源类型
* @param datasourceId 数据源ID
* @return 数据源
* @see DataSourceNotExistException
*/
default Mono<DataSource> getDataSourceOrError(DataSourceType type, String datasourceId) {
return getDataSource(type, datasourceId)
.switchIfEmpty(Mono.error(() -> new DataSourceNotExistException(type, datasourceId)));
}
/**
* 获取指定的数据源,如果数据源不存在则返回{@link Mono#empty()}
*
* @param typeId 数据源类型ID
* @param datasourceId 数据源ID
* @return 数据源
*/
Mono<DataSource> getDataSource(String typeId, String datasourceId);
/**
* 获取指定的数据源,如果数据源不存在则抛出异常{@link DataSourceNotExistException}
*
* @param typeId 数据源类型ID
* @param datasourceId 数据源ID
* @return 数据源
* @see DataSourceNotExistException
*/
default Mono<DataSource> getDataSourceOrError(String typeId, String datasourceId) {
return getDataSource(typeId, datasourceId)
.switchIfEmpty(Mono.error(() -> new DataSourceNotExistException(typeId, datasourceId)));
}
}

101
jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DataSourceProvider.java Normal file
View File

@ -0,0 +1,101 @@
package org.jetlinks.community.datasource;
import org.jetlinks.core.command.Command;
import org.jetlinks.core.command.CommandHandler;
import org.jetlinks.core.monitor.Monitor;
import reactor.core.publisher.Mono;
import javax.annotation.Nonnull;
import java.util.Map;
/**
* 数据源提供商,用于提供对数据源的支持.
*
* @author zhouhao
* @see DataSource
* @since 1.10
*/
public interface DataSourceProvider {
/**
* @return 数据源类型
*/
@Nonnull
DataSourceType getType();
/**
* 根据数据源配置来创建数据源
*
* @param properties 数据源配置
* @return 数据源
*/
@Nonnull
Mono<DataSource> createDataSource(@Nonnull DataSourceConfig properties);
/**
* 使用新的配置来重新加载数据源
*
* @param dataSource 数据源
* @param properties 配置
* @return 重新加载后的数据源
*/
@Nonnull
Mono<DataSource> reload(@Nonnull DataSource dataSource,
@Nonnull DataSourceConfig properties);
/**
* 创建命令支持，用于提供针对某个数据源的命令支持.
* <p>
* 命令执行过程中请使用{@link CommandConfiguration#getMonitor()}进行日志打印以及链路追踪。
*
* @return 命令支持
* @since 2.3
*/
default Mono<CommandHandler<?, ?>> createCommandHandler(CommandConfiguration configuration) {
return Mono.empty();
}
/**
* 命令配置
*/
interface CommandConfiguration {
/**
* 获取命令ID
*
* @return 命令ID
* @see Command#getCommandId()
*/
String getCommandId();
/**
* 获取命令名称
*
* @return 命令名称
*/
String getCommandName();
/**
* 获取命令配置信息
*
* @return 配置信息
*/
Map<String, Object> getConfiguration();
/**
* 获取数据源
*
* @return 数据源
* @see DataSource#isWrapperFor(Class)
* @see DataSource#unwrap(Class)
*/
Mono<DataSource> getDataSource();
/**
* 获取监控器,用于日志打印,链路追踪等
*
* @return 监控器
*/
Monitor getMonitor();
}
}

38
jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DataSourceState.java Normal file
View File

@ -0,0 +1,38 @@
package org.jetlinks.community.datasource;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.SneakyThrows;
@AllArgsConstructor
@Getter
public class DataSourceState {
//正常
public static String code_ok = "ok";
//正常
public static String code_stopped = "stopped";
//正常
public static String code_error = "error";
public static final DataSourceState ok = new DataSourceState(code_ok, null);
public static final DataSourceState stopped = new DataSourceState(code_stopped, null);
private final String code;
private final Throwable reason;
public boolean isOk(){
return code_ok.equals(code);
}
public static DataSourceState error(Throwable error) {
return new DataSourceState(code_error, error);
}
@SneakyThrows
public void validate() {
if (reason != null) {
throw reason;
}
}
}

7
jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DataSourceType.java Normal file
View File

@ -0,0 +1,7 @@
package org.jetlinks.community.datasource;
public interface DataSourceType {
String getId();
String getName();
}

235
jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DefaultDataSourceManager.java Normal file
View File

@ -0,0 +1,235 @@
package org.jetlinks.community.datasource;
import lombok.AllArgsConstructor;
import lombok.EqualsAndHashCode;
import lombok.Generated;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.jetlinks.core.cache.ReactiveCacheContainer;
import reactor.core.Disposable;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.util.function.Tuple2;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
/**
* 默认数据源管理器
* <p>
* 获取数据源时,如果数据源不存在.
* 则尝试从{@link DataSourceConfigManager#getConfig(String, String)}获取配置,
* 然后调用{@link DataSourceConfig#getTypeId()}对应的{@link DataSourceProvider#getType()}
* 进行初始化{@link DataSourceProvider#createDataSource(DataSourceConfig)}.
*
* <p>
* 通过实现{@link DataSourceProvider}并注入到Spring容器中,即可实现自定义数据源.
*
* <p>
* 当数据源配置发生变化时,将自动重新加载数据源.
*
* @author zhouhao
* @see DataSourceProvider
* @see DataSource
* @since 1.9
*/
@Slf4j
public class DefaultDataSourceManager implements DataSourceManager {
private final Map<String, DataSourceProvider> providers = new ConcurrentHashMap<>();
private final ReactiveCacheContainer<CacheKey, DataSource> cachedDataSources = ReactiveCacheContainer.create();
private final DataSourceConfigManager dataSourceConfigManager;
public DefaultDataSourceManager(DataSourceConfigManager configManager) {
this.dataSourceConfigManager = configManager;
this.dataSourceConfigManager
.doOnConfigChanged((state, properties) -> {
//禁用,则删除数据源
if (state == DataSourceConfigManager.ConfigState.disabled) {
this.removeDataSource(properties.getTypeId(), properties.getId());
} else {
if (cachedDataSources.containsKey(new CacheKey(properties.getTypeId(), properties.getId()))) {
//重新加载
return this
.reloadDataSource(properties.getTypeId(), properties.getId())
.then();
}
}
return Mono.empty();
});
}
/**
* 注册一个数据源提供商
*
* @param provider 数据源提供商
* @see DataSourceProvider
*/
public void register(DataSourceProvider provider) {
log.debug("Register DataSource {} Provider {}", provider.getType().getId(), provider);
providers.put(provider.getType().getId(), provider);
}
/**
* 注册一个已经初始化的数据源
*
* @param dataSource 数据源
* @see DataSource
*/
public void register(DataSource dataSource) {
log.debug("Register DataSource {} {}", dataSource.getType().getId(), dataSource);
CacheKey key = new CacheKey(dataSource.getType().getId(), dataSource.getId());
cachedDataSources.put(key, dataSource);
}
@Override
public DataSourceProvider getProvider(String typeId) {
DataSourceProvider dataSourceProvider = providers.get(typeId);
if (dataSourceProvider == null) {
throw new UnsupportedOperationException("不支持的数据源类型：" + typeId);
}
return dataSourceProvider;
}
@Override
public Flux<DataSource> getDataSources(String typeId) {
return cachedDataSources
.values()
.filter(dataSource -> Objects.equals(typeId, dataSource.getType().getId()));
}
@Override
public List<DataSourceType> getSupportedType() {
return providers
.values()
.stream()
.map(DataSourceProvider::getType)
.collect(Collectors.toList());
}
@Override
public Mono<DataSource> getDataSource(DataSourceType type,
String datasourceId) {
return getDataSource(type.getId(), datasourceId);
}
@Override
public Mono<DataSource> getDataSource(String typeId, String datasourceId) {
return getOrCreateRef(typeId, datasourceId);
}
/**
* 获取数据源引用缓存，如果没有则自动加载,如果数据源不存在,不会立即报错.
* 在使用{@link DataSourceRef#getRef()}才会返回错误.
*
* @param typeId 数据源类型ID
* @param datasourceId 数据源ID
* @return 数据源引用
*/
private Mono<DataSource> getOrCreateRef(String typeId, String datasourceId) {
return cachedDataSources
.computeIfAbsent(new CacheKey(typeId, datasourceId),
key -> loadDataSource(key.type, key.datasourceId));
}
public Mono<Tuple2<DataSourceConfig, DataSourceProvider>> loadConfigAndProvider(String typeId, String datasourceId) {
return Mono
.zip(
dataSourceConfigManager.getConfig(typeId, datasourceId),
Mono
.justOrEmpty(providers.get(typeId))
.switchIfEmpty(Mono.error(() -> new UnsupportedOperationException("unsupported datasource type " + typeId)))
);
}
public Mono<DataSource> loadDataSource(String typeId, String datasourceId) {
return this
.loadConfigAndProvider(typeId, datasourceId)
.flatMap(tp2 -> tp2.getT2().createDataSource(tp2.getT1()))
.doOnNext(dataSource -> log
.debug("load {} datasource [{}]", dataSource.getType().getId(), dataSource.getId()));
}
private void removeDataSource(String typeId, String datasourceId) {
cachedDataSources.remove(new CacheKey(typeId, datasourceId));
}
private Mono<Void> validateDataSource(DataSourceProvider provider, DataSourceConfig config) {
return provider
.createDataSource(config)
.flatMap(dataSource -> dataSource
.state()
.doOnNext(DataSourceState::validate)
//销毁测试数据源
.doAfterTerminate(dataSource::dispose)
.then()
);
}
private Mono<DataSource> reloadDataSource(String typeId, String datasourceId) {
return this
.loadConfigAndProvider(typeId, datasourceId)
//先校验一下
.flatMap(tp2 -> this
.validateDataSource(tp2.getT2(), tp2.getT1())
.thenReturn(tp2))
.flatMap(tp2 -> cachedDataSources
.compute(
new CacheKey(typeId, datasourceId),
(key, old) -> {
if (old != null) {
return tp2.getT2().reload(old, tp2.getT1());
}
return tp2.getT2().createDataSource(tp2.getT1());
}
))
.doOnError(err -> log.error("reload {} datasource [{}] error ", typeId, datasourceId, err));
}
@AllArgsConstructor
@EqualsAndHashCode
static class CacheKey {
private final String type;
private final String datasourceId;
}
static class DataSourceRef implements Disposable {
@Getter
private volatile Mono<DataSource> ref;
private boolean disposed = false;
public DataSourceRef(Mono<DataSource> ref) {
this.ref = ref;
}
@Override
public void dispose() {
ref = Mono.empty();
disposed = true;
}
@Override
@Generated
public boolean isDisposed() {
return disposed;
}
}
}

16
jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/command/CommandHandlerProvider.java Normal file
View File

@ -0,0 +1,16 @@
package org.jetlinks.community.datasource.command;
import org.jetlinks.core.command.CommandHandler;
import org.jetlinks.community.datasource.DataSourceProvider;
import org.jetlinks.community.spi.Provider;
import reactor.core.publisher.Mono;
public interface CommandHandlerProvider {
Provider<CommandHandlerProvider> supports = Provider.create(CommandHandlerProvider.class);
String getType();
Mono<CommandHandler<?, ?>> createCommandHandler(DataSourceProvider.CommandConfiguration configuration);
}

48
jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/command/DataSourceCommandConfig.java Normal file
View File

@ -0,0 +1,48 @@
package org.jetlinks.community.datasource.command;
import lombok.Getter;
import lombok.Setter;
import org.jetlinks.core.command.Command;
import org.jetlinks.community.datasource.DataSource;
import java.util.Map;
@Getter
@Setter
public class DataSourceCommandConfig {
/**
* 数据源类型
*
* @see DataSource#getType()
*/
private String datasourceType;
/**
* 数据源ID
*
* @see DataSource#getId()
*/
private String datasourceId;
/**
* 命令支持ID,比如一个数据源命令分类.
*
* @see org.jetlinks.community.command.CommandSupportManagerProvider#getCommandSupport(String, Map)
*/
private String supportId;
/**
* 命令ID
*
* @see Command#getCommandId()
*/
private String commandId;
private String commandName;
/**
* 命令配置信息
*/
private Map<String, Object> configuration;
}

238
jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/command/DataSourceCommandSupportManager.java Normal file
View File

@ -0,0 +1,238 @@
package org.jetlinks.community.datasource.command;
import lombok.AllArgsConstructor;
import org.jetlinks.core.Lazy;
import org.jetlinks.core.command.*;
import org.jetlinks.core.event.EventBus;
import org.jetlinks.core.lang.SeparatedCharSequence;
import org.jetlinks.core.lang.SharedPathString;
import org.jetlinks.core.monitor.Monitor;
import org.jetlinks.core.monitor.logger.Logger;
import org.jetlinks.core.monitor.logger.Slf4jLogger;
import org.jetlinks.core.monitor.metrics.Metrics;
import org.jetlinks.core.monitor.tracer.SimpleTracer;
import org.jetlinks.core.monitor.tracer.Tracer;
import org.jetlinks.community.command.CommandSupportManagerProvider;
import org.jetlinks.community.datasource.DataSource;
import org.jetlinks.community.datasource.DataSourceConstants;
import org.jetlinks.community.datasource.DataSourceManager;
import org.jetlinks.community.datasource.DataSourceProvider;
import org.slf4j.LoggerFactory;
import org.slf4j.event.Level;
import org.springframework.util.StringUtils;
import reactor.core.Disposable;
import reactor.core.Disposables;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
@AllArgsConstructor
public abstract class DataSourceCommandSupportManager {
public static final String defaultSupportId = "@@default";
protected final DataSourceManager dataSourceManager;
private final Map<String, DataSourceCommandSupportProvider> supports = new ConcurrentHashMap<>();
protected Mono<Void> registerCommand(DataSourceCommandConfig config) {
return supports
.computeIfAbsent(
config.getDatasourceId(),
datasourceId -> new DataSourceCommandSupportProvider(this, datasourceId))
.register(config);
}
protected Mono<Void> unregisterCommand(DataSourceCommandConfig config) {
return Mono.fromRunnable(() -> {
supports.compute(
config.getDatasourceId(),
(datasourceId,
provider) -> {
if (provider != null) {
provider.unregister(config);
if (provider.isEmpty()) {
provider.dispose();
return null;
}
}
return null;
});
});
}
protected abstract Mono<CommandSupportManagerProvider.CommandSupportInfo> getCommandSupportInfo(String datasourceId, String supportId);
static class DataSourceCommandSupportProvider
implements CommandSupportManagerProvider {
private final DataSourceCommandSupportManager parent;
private final String datasourceId;
private final Disposable.Composite disposable = Disposables.composite();
private final Map<String, DataSourceCommandSupport> commandSupports = new ConcurrentHashMap<>();
DataSourceCommandSupportProvider(DataSourceCommandSupportManager parent,
String datasourceId) {
this.datasourceId = datasourceId;
this.parent = parent;
this.disposable.add(
CommandSupportManagerProvider
.supports
.register(this.getProvider(), this)
);
}
void dispose() {
this.disposable.dispose();
}
boolean isEmpty() {
return commandSupports.isEmpty();
}
//注册命令
public Mono<Void> register(DataSourceCommandConfig config) {
return parent
.dataSourceManager
.getProvider(config.getDatasourceType())
.createCommandHandler(new CommandConfigurationImpl(config, parent.dataSourceManager))
.doOnNext(handler -> registerHandler(config, handler))
.then();
}
private void registerHandler(DataSourceCommandConfig config, CommandHandler<?, ?> handler) {
String supportId = StringUtils.hasText(config.getSupportId()) ? config.getSupportId() : defaultSupportId;
commandSupports
.computeIfAbsent(supportId,
id -> new DataSourceCommandSupport())
.registerHandler(config.getCommandId(),
handler);
}
//注销命令
public void unregister(DataSourceCommandConfig config) {
String supportId = StringUtils.hasText(config.getSupportId()) ? config.getSupportId() : defaultSupportId;
commandSupports.compute(supportId, (id, support) -> {
if (support != null) {
support.unregister(config.getCommandId());
if (support.isEmpty()) {
return null;
}
}
return null;
});
}
@Override
public String getProvider() {
return DataSourceConstants.Commands.createCommandProvider(datasourceId);
}
@Override
public Mono<? extends CommandSupport> getCommandSupport(String id, Map<String, Object> options) {
if (id == null || Objects.equals(getProvider(), id)) {
id = defaultSupportId;
}
return Mono.justOrEmpty(commandSupports.get(id));
}
@Override
public Flux<CommandSupportInfo> getSupportInfo() {
return Flux
.fromIterable(commandSupports.entrySet())
.flatMap(e -> {
String id = e.getKey();
if (Objects.equals(id, defaultSupportId)) {
id = null;
}
return parent.getCommandSupportInfo(datasourceId, id);
}, 8);
}
static class DataSourceCommandSupport extends AbstractCommandSupport {
@Override
protected <C extends Command<R>, R> void registerHandler(String id, CommandHandler<C, R> handler) {
super.registerHandler(id, handler);
}
void unregister(String commandId) {
handlers.remove(commandId);
}
boolean isEmpty() {
return handlers.isEmpty();
}
}
}
static class CommandConfigurationImpl extends Slf4jLogger
implements DataSourceProvider.CommandConfiguration, Monitor {
private final DataSourceCommandConfig config;
private final DataSourceManager manager;
private final Tracer tracer;
CommandConfigurationImpl(DataSourceCommandConfig config,
DataSourceManager manager) {
super(LoggerFactory.getLogger("org.jetlinks.community.datasource." + config.getDatasourceType()));
this.config = config;
this.manager = manager;
// TODO 企业版支持 链路追踪。
this.tracer = Tracer.noop();
}
@Override
public String getCommandId() {
return config.getCommandId();
}
@Override
public String getCommandName() {
return config.getCommandName();
}
@Override
public Map<String, Object> getConfiguration() {
return config.getConfiguration();
}
@Override
public Mono<DataSource> getDataSource() {
return manager.getDataSource(config.getDatasourceType(), config.getDatasourceId());
}
@Override
public Monitor getMonitor() {
return this;
}
@Override
public Logger logger() {
return this;
}
@Override
public Tracer tracer() {
return tracer;
}
@Override
public Metrics metrics() {
return Metrics.noop();
}
@Override
public void log(Level level, String message, Object... args) {
// TODO 企业版支持 页面中实时查看日志。
super.log(level, message, args);
}
}
}

14
jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/configuration/DataSourceHandlerProviderConfiguration.java Normal file
View File

@ -0,0 +1,14 @@
package org.jetlinks.community.datasource.configuration;
import org.springframework.boot.autoconfigure.AutoConfiguration;
import org.springframework.context.annotation.Bean;
@AutoConfiguration
public class DataSourceHandlerProviderConfiguration {
@Bean
public DataSourceHandlerProviderRegister dataSourceHandlerProviderRegister() {
return new DataSourceHandlerProviderRegister();
}
}

31
jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/configuration/DataSourceHandlerProviderRegister.java Normal file
View File

@ -0,0 +1,31 @@
package org.jetlinks.community.datasource.configuration;
import org.jetlinks.community.datasource.command.CommandHandlerProvider;
import org.springframework.beans.BeansException;
import org.springframework.beans.factory.SmartInitializingSingleton;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import javax.annotation.Nonnull;
import java.util.List;
public class DataSourceHandlerProviderRegister implements ApplicationContextAware, SmartInitializingSingleton {
private ApplicationContext context;
@Override
public void setApplicationContext(@Nonnull ApplicationContext applicationContext) throws BeansException {
this.context = applicationContext;
}
@Override
public void afterSingletonsInstantiated() {
List<CommandHandlerProvider> commandHandlerProviders = context
.getBeanProvider(CommandHandlerProvider.class)
.stream()
.toList();
commandHandlerProviders.forEach(provider -> CommandHandlerProvider.supports.register(provider.getType(), provider));
}
}

24
jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/configuration/DataSourceManagerConfiguration.java Normal file
View File

@ -0,0 +1,24 @@
package org.jetlinks.community.datasource.configuration;
import org.jetlinks.community.datasource.*;
import org.jetlinks.community.datasource.*;
import org.springframework.beans.factory.ObjectProvider;
import org.springframework.boot.autoconfigure.AutoConfiguration;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
import org.springframework.context.annotation.Bean;
@AutoConfiguration
public class DataSourceManagerConfiguration {
@Bean
@ConditionalOnBean(DataSourceConfigManager.class)
public DataSourceManager dataSourceManager(DataSourceConfigManager dataSourceConfigManager,
ObjectProvider<DataSourceProvider> providers,
ObjectProvider<DataSource> dataSources){
DefaultDataSourceManager dataSourceManager= new DefaultDataSourceManager(dataSourceConfigManager);
providers.forEach(dataSourceManager::register);
dataSources.forEach(dataSourceManager::register);
return dataSourceManager;
}
}

15
jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/exception/DataSourceNotExistException.java Normal file
View File

@ -0,0 +1,15 @@
package org.jetlinks.community.datasource.exception;
import org.hswebframework.web.exception.I18nSupportException;
import org.jetlinks.community.datasource.DataSourceType;
public class DataSourceNotExistException extends I18nSupportException {
public DataSourceNotExistException(DataSourceType datasourceType, String dataSourceId) {
this(datasourceType.getId(), dataSourceId);
}
public DataSourceNotExistException(String datasourceType, String dataSourceId) {
super("error.datasource_not_exist", datasourceType, dataSourceId);
}
}

345
jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/DefaultRDBDataSource.java Normal file
View File

@ -0,0 +1,345 @@
package org.jetlinks.community.datasource.rdb;
import com.zaxxer.hikari.HikariDataSource;
import io.r2dbc.pool.ConnectionPool;
import io.r2dbc.pool.ConnectionPoolConfiguration;
import io.r2dbc.spi.Connection;
import lombok.SneakyThrows;
import org.apache.commons.collections4.MapUtils;
import org.hswebframework.ezorm.rdb.executor.SqlRequest;
import org.hswebframework.ezorm.rdb.executor.reactive.ReactiveSyncSqlExecutor;
import org.hswebframework.ezorm.rdb.executor.wrapper.ResultWrapper;
import org.hswebframework.ezorm.rdb.metadata.RDBDatabaseMetadata;
import org.hswebframework.ezorm.rdb.metadata.RDBSchemaMetadata;
import org.hswebframework.ezorm.rdb.operator.DatabaseOperator;
import org.hswebframework.ezorm.rdb.operator.DefaultDatabaseOperator;
import org.hswebframework.web.bean.FastBeanCopier;
import org.hswebframework.web.crud.configuration.DialectProvider;
import org.hswebframework.web.crud.query.DefaultQueryHelper;
import org.hswebframework.web.crud.query.QueryHelper;
import org.hswebframework.web.crud.sql.DefaultR2dbcExecutor;
import org.hswebframework.web.exception.I18nSupportException;
import org.jetlinks.community.datasource.rdb.command.*;
import org.jetlinks.core.command.Command;
import org.jetlinks.core.command.CommandHandler;
import org.jetlinks.core.lang.SeparatedCharSequence;
import org.jetlinks.core.lang.SharedPathString;
import org.jetlinks.community.datasource.AbstractDataSource;
import org.jetlinks.community.datasource.DataSourceState;
import org.jetlinks.community.datasource.DataSourceType;
import org.jetlinks.community.datasource.rdb.command.*;
import org.jetlinks.community.utils.ObjectMappers;
import org.reactivestreams.Publisher;
import org.springframework.boot.autoconfigure.r2dbc.R2dbcProperties;
import org.springframework.boot.context.properties.PropertyMapper;
import org.springframework.boot.r2dbc.ConnectionFactoryBuilder;
import org.springframework.dao.DataAccessResourceFailureException;
import org.springframework.r2dbc.connection.ConnectionFactoryUtils;
import org.springframework.r2dbc.connection.R2dbcTransactionManager;
import org.springframework.transaction.reactive.TransactionalOperator;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.core.publisher.Sinks;
import javax.annotation.Nonnull;
import java.io.Closeable;
import java.sql.SQLException;
import java.time.Duration;
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
public class DefaultRDBDataSource extends AbstractDataSource<RDBDataSourceProperties> implements RDBDataSource {
private DatabaseOperator operator;
private String validateSql;
private final QueryHelper queryHelper;
private final List<Closeable> closeables = new CopyOnWriteArrayList<>();
private final Sinks.One<Void> loading = Sinks.one();
private TransactionalOperator transactionalOperator;
public DefaultRDBDataSource(String id,
RDBDataSourceProperties config) {
super(id, config);
init();
this.queryHelper = new DefaultQueryHelper(operator);
//刷新RDB数据源命令：Refresh
registerHandler(
Refresh.class,
CommandHandler.of(
Refresh.metadata(),
(cmd, ignore) -> cmd.execute(operator),
Refresh::new
)
);
//执行SQL命令：ExecuteSql
registerHandler(
ExecuteSql.class,
CommandHandler.of(
ExecuteSql.metadata(),
(cmd, ignore) -> loading
.asMono()
.thenMany(cmd.execute(operator)),
ExecuteSql::new
)
);
//执行列表查询命令：QueryList
registerHandler(
QueryList.class,
CommandHandler.of(
QueryList.metadata(),
(cmd, ignore) -> loading
.asMono()
.thenMany(cmd.execute(operator)),
QueryList::new
)
);
//执行分页查询命令：QueryPager
registerHandler(
QueryPager.class,
CommandHandler.of(
QueryPager.metadata(),
(cmd, ignore) -> loading
.asMono()
.then(cmd.execute(operator)),
QueryPager::new
)
);
//执行统计数量命令：Count
registerHandler(
Count.class,
CommandHandler.of(
Count.metadata(),
(cmd, ignore) -> loading
.asMono()
.then(cmd.execute(operator)),
Count::new
)
);
}
private void loadTables() {
new Refresh().execute(operator)
.doOnTerminate(loading::tryEmitEmpty)
.subscribe();
}
@Override
public DatabaseOperator operator() {
return operator;
}
@Override
public QueryHelper helper() {
return queryHelper;
}
@Override
public DataSourceType getType() {
return RDBDataSourceType.rdb;
}
@SneakyThrows
public void init() {
try {
if (getConfig().getType() == RDBDataSourceProperties.Type.r2dbc) {
initR2dbc();
} else {
initJdbc();
}
loadTables();
validateSql = getConfig().getValidateQuery();
} catch (Throwable e) {
throw translateException(e);
}
}
@SneakyThrows
synchronized void initR2dbc() {
DialectProvider dialect = getConfig().dialectProvider();
R2dbcProperties properties;
if (MapUtils.isNotEmpty(getConfig().getOthers())) {
//使用jsonCopy,FastBeanCopier不支持final字段copy.
properties = ObjectMappers
.parseJson(ObjectMappers.toJsonBytes(getConfig().getOthers()), R2dbcProperties.class);
} else {
properties = new R2dbcProperties();
}
properties.setUrl(getConfig().getUrl());
properties.setUsername(getConfig().getUsername());
properties.setPassword(getConfig().getPassword());
PropertyMapper mapper = PropertyMapper.get().alwaysApplyingWhenNonNull();
ConnectionFactoryBuilder connectionFactoryBuilder = ConnectionFactoryBuilder.withUrl(properties.getUrl());
mapper.from(properties.getUsername()).whenNonNull().to(connectionFactoryBuilder::username);
mapper.from(properties.getPassword()).whenNonNull().to(connectionFactoryBuilder::password);
R2dbcProperties.Pool pool = properties.getPool();
ConnectionPoolConfiguration.Builder builder = ConnectionPoolConfiguration.builder(connectionFactoryBuilder.build());
builder.maxLifeTime(Duration.ofMinutes(10));
builder.maxAcquireTime(Duration.ofSeconds(10));
mapper.from(pool.getMaxIdleTime()).to(builder::maxIdleTime);
mapper.from(pool.getInitialSize()).to(builder::initialSize);
mapper.from(pool.getMaxSize()).to(builder::maxSize);
mapper.from(pool.getValidationQuery()).whenHasText().to(builder::validationQuery);
mapper.from(pool.getMaxLifeTime()).whenNonNull().to(builder::maxLifeTime);
mapper.from(pool.getMaxAcquireTime()).whenNonNull().to(builder::maxAcquireTime);
mapper.from(pool.getMaxLifeTime()).whenNonNull().to(builder::maxLifeTime);
mapper.from(pool.getMaxAcquireTime()).whenNonNull().to(builder::maxAcquireTime);
mapper.from(pool.getValidationQuery()).whenHasText().to(builder::validationQuery);
ConnectionPool connectionPool = new ConnectionPool(builder.build());
closeables.add(() -> connectionPool.close().subscribe());
transactionalOperator = TransactionalOperator.create(new R2dbcTransactionManager(connectionPool));
DefaultR2dbcExecutor executor = new DefaultR2dbcExecutor() {
@Override
protected Mono<Connection> getConnection() {
return ConnectionFactoryUtils.getConnection(connectionPool);
}
@Override
public Mono<Void> execute(Publisher<SqlRequest> request) {
return super
.execute(request)
.as(transactionalOperator::transactional);
}
@Override
public Mono<Integer> update(Publisher<SqlRequest> request) {
return super
.update(request)
.as(transactionalOperator::transactional);
}
@Override
public <E> Flux<E> select(Publisher<SqlRequest> request, ResultWrapper<E, ?> wrapper) {
return super
.select(request, wrapper)
.as(transactionalOperator::transactional);
}
};
executor.setBindSymbol(dialect.getBindSymbol());
executor.setBindCustomSymbol(!executor.getBindSymbol().equals("?"));
RDBDatabaseMetadata database = new RDBDatabaseMetadata(dialect.getDialect());
database.addFeature(executor);
database.addFeature(ReactiveSyncSqlExecutor.of(executor));
RDBSchemaMetadata schema = dialect.createSchema(getConfig().getSchema());
database.addSchema(schema);
database.setCurrentSchema(schema);
this.operator = DefaultDatabaseOperator.of(database);
}
private Throwable translateException(Throwable err) {
if (err instanceof IllegalStateException) {
if (err.getMessage() != null && err.getMessage().contains("Available drivers")) {
return new I18nSupportException("error.unsupported_database_type", err);
}
}
if (err instanceof DataAccessResourceFailureException) {
return new I18nSupportException("error.database_access_error", err);
}
if (err instanceof SQLException) {
String msg = err.getMessage();
if (msg.contains("No suitable driver")) {
return new I18nSupportException("error.unsupported_database_type", err);
}
return err;
}
if (err.getClass() == RuntimeException.class) {
if (err.getCause() != null && err.getCause() != err) {
return translateException(err.getCause());
}
}
return err;
}
synchronized void initJdbc() {
HikariDataSource dataSource = new HikariDataSource();
dataSource.setJdbcUrl(getConfig().getUrl());
dataSource.setUsername(getConfig().getUsername());
dataSource.setPassword(getConfig().getPassword());
if (MapUtils.isNotEmpty(getConfig().getOthers())) {
FastBeanCopier.copy(getConfig().getOthers(), dataSource);
}
closeables.add(dataSource);
RDBDatabaseMetadata database = new RDBDatabaseMetadata(getConfig().dialectProvider().getDialect());
database.addFeature(new RDBJdbcReactiveSqlExecutor(dataSource));
database.addFeature(new RDBJdbcSyncSqlExecutor(dataSource));
RDBSchemaMetadata schema = getConfig().dialectProvider().createSchema(getConfig().getSchema());
database.addSchema(schema);
database.setCurrentSchema(schema);
this.operator = DefaultDatabaseOperator.of(database);
this.transactionalOperator = null;
}
@Override
protected Mono<DataSourceState> checkState() {
return operator
.sql()
.reactive()
.select(validateSql)
.map(i -> DataSourceState.ok)
.onErrorResume(err -> Mono.just(DataSourceState.error(translateException(err))))
.last();
}
@Override
@SuppressWarnings("all")
protected <R> R executeUndefinedCommand(@Nonnull Command<R> command) {
if (command instanceof RDBCommand) {
R r = ((RDBCommand<R>) command).execute(operator);
if (transactionalOperator != null) {
if (r instanceof Mono) {
return (R) transactionalOperator.transactional(((Mono) r));
} else if (r instanceof Flux) {
return (R) transactionalOperator.transactional(((Flux) r));
}
}
return r;
}
return super.executeUndefinedCommand(command);
}
@Override
protected void handleSetConfig(RDBDataSourceProperties oldConfig,
RDBDataSourceProperties newConfig) {
if (oldConfig == null) {
return;
}
releaseOld();
init();
}
protected void releaseOld() {
closeables.removeIf(closeable -> {
try {
closeable.close();
} catch (Throwable ignore) {
}
return true;
});
}
@Override
protected void doOnDispose() {
releaseOld();
}
}

33
jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/RDBDataSource.java Normal file
View File

@ -0,0 +1,33 @@
package org.jetlinks.community.datasource.rdb;
import org.hswebframework.ezorm.rdb.operator.DatabaseOperator;
import org.hswebframework.web.crud.query.QueryHelper;
import org.jetlinks.community.datasource.rdb.command.*;
import org.jetlinks.community.datasource.DataSource;
import org.jetlinks.community.datasource.DataSourceType;
import org.jetlinks.core.command.Command;
import javax.annotation.Nonnull;
public interface RDBDataSource extends DataSource {
RDBDataSourceProperties getConfig();
RDBDataSourceProperties copyConfig();
DatabaseOperator operator();
QueryHelper helper();
@Nonnull
@Override
default <R> R execute(@Nonnull Command<R> command) {
return DataSource.super.execute(command);
}
@Override
default DataSourceType getType() {
return RDBDataSourceType.rdb;
}
}

93
jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/RDBDataSourceProperties.java Normal file
View File

@ -0,0 +1,93 @@
package org.jetlinks.community.datasource.rdb;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.AccessLevel;
import lombok.Getter;
import lombok.Setter;
import org.hswebframework.web.crud.configuration.DialectProvider;
import org.hswebframework.web.crud.configuration.DialectProviders;
import org.hswebframework.web.crud.configuration.EasyormProperties;
import org.hswebframework.web.exception.ValidationException;
import org.hswebframework.web.validator.ValidatorUtils;
import jakarta.validation.constraints.NotBlank;
import java.net.URI;
import java.util.Map;
@Getter
@Setter
public class RDBDataSourceProperties {
private Type type;
@Schema(description = "url")
@NotBlank
private String url;
@NotBlank
@Schema(description = "数据库")
private String schema;
@Schema(description = "用户名")
private String username;
@Schema(description = "密码")
private String password;
private Map<String, Object> others;
@Schema(description = "数据库方言")
private String dialect;
@Getter(AccessLevel.PRIVATE)
@Setter(AccessLevel.PRIVATE)
private transient DialectProvider dialectProvider;
public String getValidateQuery() {
return dialectProvider().getValidationSql();
}
public DialectProvider dialectProvider() {
URI uri = URI.create(getUrl());
if (null == dialectProvider) {
if (null != dialect) {
dialectProvider = DialectProviders.lookup(dialect);
} else if (getUrl().contains("mysql") || getUrl().contains("mariadb")) {
return EasyormProperties.DialectEnum.mysql;
} else if (getUrl().contains("postgresql")) {
return EasyormProperties.DialectEnum.postgres;
} else if (getUrl().contains("oracle")) {
return EasyormProperties.DialectEnum.oracle;
} else if (getUrl().contains("mssql") || getUrl().contains("sqlserver")) {
return EasyormProperties.DialectEnum.mssql;
} else if (getUrl().contains("h2")) {
return EasyormProperties.DialectEnum.h2;
} else if (getUrl().contains("dm")) {
return DialectProviders.lookup("dm");
} else {
throw new ValidationException("url", "error.unsupported_database_type", uri.getFragment());
}
}
return dialectProvider;
}
public RDBDataSourceProperties validate() {
ValidatorUtils.tryValidate(this);
return this;
}
public Type getType() {
if (type == null) {
type = url.startsWith("r2dbc") ? Type.r2dbc : Type.jdbc;
}
return type;
}
public enum Type {
jdbc,
r2dbc;
}
}

217
jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/RDBDataSourceProvider.java Normal file
View File

@ -0,0 +1,217 @@
package org.jetlinks.community.datasource.rdb;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import org.hswebframework.ezorm.rdb.metadata.RDBColumnMetadata;
import org.hswebframework.ezorm.rdb.operator.DatabaseOperator;
import org.hswebframework.web.api.crud.entity.QueryParamEntity;
import org.hswebframework.web.bean.FastBeanCopier;
import org.hswebframework.web.crud.query.DefaultQueryHelper;
import org.hswebframework.web.crud.query.QueryAnalyzer;
import org.hswebframework.web.crud.query.QueryHelper;
import org.jetlinks.community.datasource.DataSource;
import org.jetlinks.community.datasource.DataSourceConfig;
import org.jetlinks.community.datasource.DataSourceProvider;
import org.jetlinks.community.datasource.DataSourceType;
import org.jetlinks.core.command.CommandHandler;
import org.jetlinks.core.metadata.DataType;
import org.jetlinks.core.metadata.PropertyMetadata;
import org.jetlinks.core.metadata.SimplePropertyMetadata;
import org.jetlinks.core.metadata.types.*;
import org.jetlinks.core.monitor.Monitor;
import org.jetlinks.community.datasource.*;
import org.jetlinks.community.datasource.rdb.command.QueryList;
import org.jetlinks.community.datasource.rdb.command.QueryPager;
import org.jetlinks.community.datasource.rdb.command.RDBRequestListCommand;
import org.jetlinks.community.datasource.rdb.command.RDBRequestPagerCommand;
import org.jetlinks.community.things.utils.ThingsDatabaseUtils;
import org.jetlinks.sdk.server.commons.cmd.QueryPagerCommand;
import org.springframework.stereotype.Component;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
import javax.annotation.Nonnull;
import java.util.*;
import java.util.function.Function;
@Component
public class RDBDataSourceProvider implements DataSourceProvider {
@Nonnull
@Override
public DataSourceType getType() {
return RDBDataSourceType.rdb;
}
@Nonnull
@Override
public Mono<DataSource> createDataSource(@Nonnull DataSourceConfig properties) {
return Mono
.<DataSource>fromCallable(() -> RDBDataSourceProvider
.create(properties.getId(),
FastBeanCopier.copy(properties.getConfiguration(), new RDBDataSourceProperties())
.validate()
)
)
.subscribeOn(Schedulers.boundedElastic());
}
@Nonnull
@Override
public Mono<DataSource> reload(@Nonnull DataSource dataSource, @Nonnull DataSourceConfig properties) {
return Mono
.defer(() -> {
RDBDataSourceProperties dataSourceProperties = FastBeanCopier
.copy(properties.getConfiguration(), RDBDataSourceProperties::new)
.validate();
if (dataSource.isWrapperFor(DefaultRDBDataSource.class)) {
dataSource
.unwrap(DefaultRDBDataSource.class)
.setConfig(dataSourceProperties);
return Mono.just(dataSource);
}
dataSource.dispose();
return createDataSource(properties);
})
.subscribeOn(Schedulers.boundedElastic());
}
public static RDBDataSource create(String id, RDBDataSourceProperties properties) {
return new DefaultRDBDataSource(id, properties);
}
@Override
public Mono<CommandHandler<?, ?>> createCommandHandler(CommandConfiguration configuration) {
Configuration config = FastBeanCopier.copy(configuration.getConfiguration(), Configuration.class);
RDBDefinition rdbDefinition = config.getRdbDefinition();
Boolean paging = rdbDefinition.getPaging();
return getRdbDataSource(configuration)
.flatMap(source -> {
List<PropertyMetadata> resultColumns = getResultColumns(
rdbDefinition.getSql(),
source.helper(),
configuration.getMonitor());
if (paging) {
return Mono.just(
RDBRequestPagerCommand
.createQueryHandler(
configuration.getCommandId(),
configuration.getCommandName(),
metadata -> metadata.setOutput(QueryPagerCommand.createOutputType(resultColumns)),
cmd -> getRdbDataSource(configuration)
.flatMap(database -> database.execute(new QueryPager().with(cmd.with("sql", rdbDefinition.getSql()).asMap())))
)
);
}
return Mono.just(
RDBRequestListCommand
.createQueryHandler(
configuration.getCommandId(),
configuration.getCommandName(),
metadata -> metadata.setOutput(getResultType(resultColumns)),
cmd -> getRdbDataSource(configuration)
.flatMapMany(database -> database.execute(new QueryList().with(cmd.with("sql", rdbDefinition.getSql()).asMap())))
));
});
}
private static Mono<RDBDataSource> getRdbDataSource(CommandConfiguration configuration) {
return configuration
.getDataSource()
.map(datasource -> datasource.unwrap(RDBDataSource.class));
}
private Flux<Map<String, Object>> queryList(String sql, DatabaseOperator operator, QueryParamEntity param) {
QueryHelper queryHelper = new DefaultQueryHelper(operator);
return queryHelper
.select(sql)
.where(param)
.fetch()
.map(Function.identity());
}
private DataType getResultType(List<PropertyMetadata> propertyMetadata) {
ObjectType type = new ObjectType();
type.setProperties(propertyMetadata);
return type;
}
private List<PropertyMetadata> getResultColumns(String sql, QueryHelper queryHelper, Monitor monitor) {
try {
QueryAnalyzer _analyzer = queryHelper.analysis(sql);
return parseColumToProperties(_analyzer);
} catch (Exception e) {
monitor
.logger()
.error("初始化sql查询失败", e);
}
return Collections.emptyList();
}
static List<PropertyMetadata> parseColumToProperties(QueryAnalyzer analyzer) {
List<PropertyMetadata> columns = new ArrayList<>();
QueryAnalyzer.Table table = analyzer.select().getTable();
Map<String, List<PropertyMetadata>> nests = new LinkedHashMap<>();
for (QueryAnalyzer.Column column : analyzer.select().getColumnList()) {
DataType type;
String name;
RDBColumnMetadata metadata = column.getMetadata();
if (metadata == null) {
type = StringType.GLOBAL;
name = column.getName();
} else {
type = ThingsDatabaseUtils.convertDataType(column.getMetadata());
name = column.getMetadata().getComment();
}
if (Objects.equals(column.getOwner(), table.getAlias())) {
columns.add(SimplePropertyMetadata.of(column.getAlias(), name, type));
} else {
nests.computeIfAbsent(column.getOwner(), __ -> new ArrayList<>())
.add(SimplePropertyMetadata.of(column.getAlias(), name, type));
}
}
for (Map.Entry<String, List<PropertyMetadata>> nest : nests.entrySet()) {
ObjectType type = new ObjectType();
type.setProperties(nest.getValue());
columns.add(SimplePropertyMetadata.of(nest.getKey(), nest.getKey(), type));
}
return columns;
}
@Getter
@Setter
public static class Configuration {
private RDBDefinition rdbDefinition;
}
@Getter
@Setter
@AllArgsConstructor
@NoArgsConstructor
public static class RDBDefinition {
@Schema(description = "sql语句")
private String sql;
@Schema(description = "是否分页")
private Boolean paging;
@Schema(description = "其他配置")
private Map<String, Object> others;
}
}

18
jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/RDBDataSourceType.java Normal file
View File

@ -0,0 +1,18 @@
package org.jetlinks.community.datasource.rdb;
import org.jetlinks.community.datasource.DataSourceType;
public enum RDBDataSourceType implements DataSourceType {
rdb
;
@Override
public String getId() {
return name();
}
@Override
public String getName() {
return "关系型数据库";
}
}

58
jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/RDBJdbcReactiveSqlExecutor.java Normal file
View File

@ -0,0 +1,58 @@
package org.jetlinks.community.datasource.rdb;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.hswebframework.ezorm.rdb.executor.SqlRequest;
import org.hswebframework.ezorm.rdb.executor.jdbc.JdbcReactiveSqlExecutor;
import org.hswebframework.ezorm.rdb.executor.wrapper.ResultWrapper;
import org.reactivestreams.Publisher;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
import javax.sql.DataSource;
import java.sql.Connection;
import java.sql.SQLException;
@AllArgsConstructor
@Slf4j
public class RDBJdbcReactiveSqlExecutor extends JdbcReactiveSqlExecutor {
private final DataSource dataSource;
@Override
public Mono<Connection> getConnection() {
return Mono
.using(dataSource::getConnection,
Mono::just,
source -> {
try {
source.close();
} catch (SQLException e) {
log.error(e.getMessage(), e);
}
},
false
);
}
@Override
public Mono<Void> execute(Publisher<SqlRequest> request) {
return super
.execute(request)
.subscribeOn(Schedulers.boundedElastic());
}
@Override
public Mono<Integer> update(Publisher<SqlRequest> request) {
return super
.update(request)
.subscribeOn(Schedulers.boundedElastic());
}
@Override
public <E> Flux<E> select(Publisher<SqlRequest> request, ResultWrapper<E, ?> wrapper) {
return super
.select(request, wrapper)
.subscribeOn(Schedulers.boundedElastic());
}
}

26
jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/RDBJdbcSyncSqlExecutor.java Normal file
View File

@ -0,0 +1,26 @@
package org.jetlinks.community.datasource.rdb;
import lombok.AllArgsConstructor;
import lombok.SneakyThrows;
import org.hswebframework.ezorm.rdb.executor.SqlRequest;
import org.hswebframework.ezorm.rdb.executor.jdbc.JdbcSyncSqlExecutor;
import javax.sql.DataSource;
import java.sql.Connection;
@AllArgsConstructor
public class RDBJdbcSyncSqlExecutor extends JdbcSyncSqlExecutor {
private final DataSource dataSource;
@Override
@SneakyThrows
public Connection getConnection(SqlRequest sqlRequest) {
return dataSource.getConnection();
}
@Override
@SneakyThrows
public void releaseConnection(Connection connection, SqlRequest sqlRequest) {
connection.close();
}
}

39
jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/Column.java Normal file
View File

@ -0,0 +1,39 @@
package org.jetlinks.community.datasource.rdb.command;
import lombok.*;
import org.hswebframework.ezorm.rdb.metadata.RDBColumnMetadata;
@Getter
@Setter
@AllArgsConstructor
@NoArgsConstructor
@Builder
public class Column {
private String name;
private String previousName;
private String type;
private boolean primaryKey;
private int length;
private int scale;
private int precision;
private boolean notnull;
private String comment;
public static Column of(RDBColumnMetadata columnMetadata) {
Column column = new Column();
column.setPreviousName(columnMetadata.getName());
column.setName(columnMetadata.getName());
column.setType(columnMetadata.getType().getSqlType().getName());
column.setLength(columnMetadata.getLength() == 0 ? columnMetadata.getPrecision() : columnMetadata.getLength());
column.setScale(columnMetadata.getScale());
column.setPrecision(columnMetadata.getPrecision());
column.setNotnull(columnMetadata.isNotNull());
column.setPrimaryKey(columnMetadata.isPrimaryKey());
column.setComment(columnMetadata.getComment());
return column;
}
}

69
jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/Count.java Normal file
View File

@ -0,0 +1,69 @@
package org.jetlinks.community.datasource.rdb.command;
import org.hswebframework.ezorm.rdb.executor.wrapper.MapResultWrapper;
import org.hswebframework.ezorm.rdb.operator.DatabaseOperator;
import org.hswebframework.web.api.crud.entity.QueryParamEntity;
import org.hswebframework.web.crud.query.DefaultQueryHelper;
import org.hswebframework.web.exception.BusinessException;
import org.jetlinks.core.metadata.FunctionMetadata;
import org.jetlinks.core.metadata.PropertyMetadata;
import org.jetlinks.community.datasource.DataSourceConstants;
import org.jetlinks.sdk.server.commons.cmd.CountCommand;
import org.springframework.data.util.Pair;
import reactor.core.publisher.Mono;
import java.util.List;
import java.util.Optional;
public class Count extends CountCommand implements RDBCommand<Mono<Integer>> {
public Pair<String, String> getSqlOrTable() {
return Optional.ofNullable(readable().get("sql"))
.map(sql -> Pair.of("sql", String.valueOf(sql)))
.orElse(Optional.ofNullable(readable().get("table"))
.map(table -> Pair.of("table", String.valueOf(table)))
.orElse(null));
}
@Override
public Mono<Integer> execute(DatabaseOperator operator) {
if (getSqlOrTable() == null) {
return Mono.error(new UnsupportedOperationException("sql or table is not found"));
}
QueryParamEntity param = this.asQueryParam();
param.setPaging(false);
DefaultQueryHelper queryHelper = new DefaultQueryHelper(operator);
if (getSqlOrTable().getFirst().equals("sql")) {
return queryHelper
.select(getSqlOrTable().getSecond())
.where(param)
.count();
}
if (getSqlOrTable().getFirst().equals("table")) {
return operator
.dml()
.query(getSqlOrTable().getSecond())
.setParam(param)
.fetch(new MapResultWrapper())
.reactive()
.count()
.map(Long::intValue)
.onErrorResume(err -> Mono.error(new BusinessException(err.getMessage())));
}
return Mono.empty();
}
public static FunctionMetadata metadata() {
List<PropertyMetadata> list = QueryList.getInputList();
return DataSourceConstants.Metadata
.create(Count.class, func -> {
func.setName("统计数量");
func.setInputs(list);
});
}
}

39
jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/CreateOrAlterTable.java Normal file
View File

@ -0,0 +1,39 @@
package org.jetlinks.community.datasource.rdb.command;
import lombok.AllArgsConstructor;
import org.hswebframework.ezorm.rdb.operator.DatabaseOperator;
import org.hswebframework.ezorm.rdb.operator.ddl.TableBuilder;
import reactor.core.publisher.Mono;
@AllArgsConstructor
public class CreateOrAlterTable implements RDBCommand<Mono<Void>> {
private final Table table;
@Override
public Mono<Void> execute(DatabaseOperator operator) {
TableBuilder builder = operator
.ddl()
.createOrAlter(table.getName());
for (Column column : table.getColumns()) {
builder
.addColumn(column.getName())
.custom(rdb -> {
rdb.setPrimaryKey(column.isPrimaryKey());
rdb.setNotNull(column.isNotnull());
rdb.setPreviousName(column.getPreviousName());
})
.length(column.getLength(), column.getScale())
.type(column.getType())
.comment(column.getComment())
.commit();
}
return builder
.commit()
.reactive()
.then();
}
}

27
jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/DropColumn.java Normal file
View File

@ -0,0 +1,27 @@
package org.jetlinks.community.datasource.rdb.command;
import lombok.AllArgsConstructor;
import org.hswebframework.ezorm.rdb.operator.DatabaseOperator;
import org.hswebframework.ezorm.rdb.operator.ddl.TableBuilder;
import reactor.core.publisher.Mono;
import java.util.Set;
@AllArgsConstructor
public class DropColumn implements RDBCommand<Mono<Void>> {
private final String table;
private final Set<String> columns;
@Override
public Mono<Void> execute(DatabaseOperator operator) {
TableBuilder builder = operator
.ddl()
.createOrAlter(table);
columns.forEach(builder::dropColumn);
return builder
.commit()
.reactive()
.then();
}
}

206
jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/ExecuteSql.java Normal file
View File

@ -0,0 +1,206 @@
package org.jetlinks.community.datasource.rdb.command;
import lombok.*;
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
import net.sf.jsqlparser.statement.Statement;
import net.sf.jsqlparser.statement.select.Select;
import org.hswebframework.ezorm.rdb.executor.SqlRequest;
import org.hswebframework.ezorm.rdb.executor.SqlRequests;
import org.hswebframework.ezorm.rdb.executor.wrapper.ColumnWrapperContext;
import org.hswebframework.ezorm.rdb.executor.wrapper.ResultWrapper;
import org.hswebframework.ezorm.rdb.operator.DatabaseOperator;
import org.hswebframework.web.bean.FastBeanCopier;
import org.hswebframework.web.crud.query.DefaultQueryHelper;
import org.hswebframework.web.crud.query.QueryAnalyzer;
import org.jetlinks.community.utils.ConverterUtils;
import org.jetlinks.core.command.AbstractCommand;
import org.jetlinks.core.metadata.*;
import org.jetlinks.core.metadata.types.ArrayType;
import org.jetlinks.core.metadata.types.ObjectType;
import org.jetlinks.core.metadata.types.StringType;
import org.jetlinks.community.ConfigMetadataConstants;
import org.jetlinks.community.datasource.DataSourceConstants;
import org.springframework.util.Assert;
import reactor.core.publisher.Flux;
import java.io.Serializable;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Consumer;
public class ExecuteSql extends AbstractCommand<Flux<ExecuteSql.SqlResult<?>>, ExecuteSql>
implements RDBCommand<Flux<ExecuteSql.SqlResult<?>>> {
private static final String PARAMETERSTRING = "sqlRequests";
public List<ExecuteSqlRequest> getSqlRequests() {
return ConverterUtils.convertToList(readable().getOrDefault(PARAMETERSTRING, new ExecuteSqlRequest()),
params -> FastBeanCopier.copy(params, new ExecuteSqlRequest()));
}
@Override
public Flux<SqlResult<?>> execute(DatabaseOperator operator) {
return Flux.fromIterable(getSqlRequests())
.flatMap(sqlRequest -> {
String sql = sqlRequest.getSql();
Assert.hasText(sql, "'sql' can not be empty");
SqlRequest finalRealSql = SqlRequests.template(sql, sqlRequest.getParameter());
Statement statement = parseSql(finalRealSql.getSql());
if (statement instanceof Select) {
return operator
.sql()
.reactive()
.select(finalRealSql, new SqlResultWrapper())
.defaultIfEmpty(parseSelect(finalRealSql.getSql(), operator))
.take(1000)
.collectList()
.map(list -> SqlResult.of(list, Operation.select.name()));
}
return operator
.sql()
.reactive()
.update(finalRealSql)
.map(updateCount -> SqlResult.of(updateCount, Operation.upsert.name()));
});
}
@AllArgsConstructor
static class SqlResultWrapper implements ResultWrapper<LinkedHashMap<String, Object>, Void> {
private final Map<String, Integer> columnCountMap = new ConcurrentHashMap<>();
@Override
public LinkedHashMap<String, Object> newRowInstance() {
return new LinkedHashMap<>();
}
@Override
public void wrapColumn(ColumnWrapperContext<LinkedHashMap<String, Object>> context) {
String column = context.getColumnLabel();
Object value = String.valueOf(context.getResult());
columnCountMap.compute(column, (key, count) -> {
if (context.getRowInstance().containsKey(key)) {
count = (count == null) ? 1 : count + 1;
StringBuilder sb = new StringBuilder(column)
.append("(")
.append(count)
.append(")");
context.getRowInstance().put(sb.toString(), value);
} else {
context.getRowInstance().put(key, value);
count = 0;
}
return count;
});
}
@Override
public boolean completedWrapRow(LinkedHashMap<String, Object> result) {
if (!columnCountMap.isEmpty()) {
columnCountMap.clear();
}
return true;
}
@Override
public Void getResult() {
return null;
}
}
private LinkedHashMap<String, Object> parseSelect(String sql, DatabaseOperator operator) {
QueryAnalyzer.Select select = new DefaultQueryHelper(operator)
.analysis(sql)
.select();
LinkedHashMap<String, Object> result = new LinkedHashMap<>();
select.getColumnList().forEach(column -> result.put(column.getName(), "(N/A)"));
return result;
}
@SneakyThrows
public Statement parseSql(String sql) {
return CCJSqlParserUtil.parse(sql);
}
public static FunctionMetadata metadata(Consumer<SimpleFunctionMetadata> custom) {
SimpleFunctionMetadata metadata = new SimpleFunctionMetadata();
//ExecuteSql
metadata.setId("ExecuteSql");
metadata.setName("执行SQL语句");
metadata.setInputs(Collections.singletonList(
SimplePropertyMetadata
.of(
PARAMETERSTRING,
"SQL查询参数",
new ArrayType().elementType(
new ObjectType()
.addProperty("sql", "sql语句", new StringType().expand(ConfigMetadataConstants.required,
true))
.addProperty("parameter", "替换参数", new ObjectType()))
)
));
custom.accept(metadata);
return metadata;
}
public static FunctionMetadata metadata() {
return DataSourceConstants.Metadata
.create(ExecuteSql.class, func -> {
func.setName("执行SQL语句");
func.setInputs(Collections.singletonList(
SimplePropertyMetadata
.of(
PARAMETERSTRING,
"SQL查询参数",
new ArrayType().elementType(
new ObjectType()
.addProperty("sql", "sql语句", new StringType().expand(ConfigMetadataConstants.required,
true))
.addProperty("parameter", "替换参数", new ObjectType()))
)
));
});
}
private enum Operation {
select,
upsert
}
@Getter
@Setter
@AllArgsConstructor(staticName = "of")
public static class SqlResult<T> {
private T data;
private String type;
}
@Getter
@Setter
@AllArgsConstructor
@NoArgsConstructor
public static class ExecuteSqlRequest implements Serializable {
private String sql;
private Object parameter;
}
}

24
jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/GetTable.java Normal file
View File

@ -0,0 +1,24 @@
package org.jetlinks.community.datasource.rdb.command;
import lombok.AllArgsConstructor;
import org.hswebframework.ezorm.rdb.metadata.TableOrViewMetadata;
import org.hswebframework.ezorm.rdb.metadata.parser.TableMetadataParser;
import org.hswebframework.ezorm.rdb.operator.DatabaseOperator;
import reactor.core.publisher.Mono;
@AllArgsConstructor
public class GetTable implements RDBCommand<Mono<Table>> {
private final String name;
@Override
public Mono<Table> execute(DatabaseOperator operator) {
return operator
.getMetadata()
.getCurrentSchema()
.<TableMetadataParser>findFeatureNow(TableMetadataParser.id)
.parseByNameReactive(name)
.cast(TableOrViewMetadata.class)
.map(Table::of);
}
}

29
jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/GetTables.java Normal file
View File

@ -0,0 +1,29 @@
package org.jetlinks.community.datasource.rdb.command;
import lombok.AllArgsConstructor;
import lombok.NoArgsConstructor;
import org.hswebframework.ezorm.rdb.metadata.TableOrViewMetadata;
import org.hswebframework.ezorm.rdb.metadata.parser.TableMetadataParser;
import org.hswebframework.ezorm.rdb.operator.DatabaseOperator;
import reactor.core.publisher.Flux;
@NoArgsConstructor
@AllArgsConstructor
public class GetTables implements RDBCommand<Flux<Table>> {
private boolean includeColumns;
@Override
public Flux<Table> execute(DatabaseOperator operator) {
TableMetadataParser parser = operator
.getMetadata()
.getCurrentSchema()
.findFeatureNow(TableMetadataParser.id);
return
includeColumns
? parser.parseAllReactive().cast(TableOrViewMetadata.class).map(Table::of)
: parser.parseAllTableNameReactive().map(name -> new Table(name, null));
}
}

77
jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/QueryList.java Normal file
View File

@ -0,0 +1,77 @@
package org.jetlinks.community.datasource.rdb.command;
import org.hswebframework.ezorm.rdb.executor.wrapper.MapResultWrapper;
import org.hswebframework.ezorm.rdb.operator.DatabaseOperator;
import org.hswebframework.web.api.crud.entity.QueryParamEntity;
import org.hswebframework.web.crud.query.DefaultQueryHelper;
import org.jetlinks.core.metadata.FunctionMetadata;
import org.jetlinks.core.metadata.PropertyMetadata;
import org.jetlinks.core.metadata.SimplePropertyMetadata;
import org.jetlinks.core.metadata.types.StringType;
import org.jetlinks.community.datasource.DataSourceConstants;
import org.jetlinks.sdk.server.commons.cmd.QueryListCommand;
import org.springframework.data.util.Pair;
import reactor.core.publisher.Flux;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.Function;
public class QueryList extends QueryListCommand<Map<String, Object>>
implements RDBCommand<Flux<Map<String, Object>>>{
public Pair<String, String> getSqlOrTable() {
return Optional.ofNullable(readable().get("sql"))
.map(sql -> Pair.of("sql", String.valueOf(sql)))
.orElse(Optional.ofNullable(readable().get("table"))
.map(table -> Pair.of("table", String.valueOf(table)))
.orElse(null));
}
@Override
public Flux<Map<String, Object>> execute(DatabaseOperator operator) {
if (getSqlOrTable() == null) {
return Flux.error(new UnsupportedOperationException("sql or table is not found"));
}
QueryParamEntity param = this.asQueryParam();
DefaultQueryHelper queryHelper = new DefaultQueryHelper(operator);
if (getSqlOrTable().getFirst().equals("sql")) {
return queryHelper
.select(getSqlOrTable().getSecond())
.where(param)
.fetch()
.map(Function.identity());
}
if (getSqlOrTable().getFirst().equals("table")) {
return operator
.dml()
.query(getSqlOrTable().getSecond())
.setParam(param)
.fetch(new MapResultWrapper())
.reactive();
}
return Flux.empty();
}
public static FunctionMetadata metadata() {
List<PropertyMetadata> list = getInputList();
return DataSourceConstants.Metadata
.create(QueryList.class, func -> {
func.setName("列表查询");
func.setInputs(list);
});
}
public static List<PropertyMetadata> getInputList() {
List<PropertyMetadata> list = new ArrayList<>(QueryListCommand.getQueryParamMetadata());
list.add(SimplePropertyMetadata.of("table", "表名", StringType.GLOBAL));
list.add(SimplePropertyMetadata.of("sql", "sql语句", StringType.GLOBAL));
return list;
}
}

121
jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/QueryPager.java Normal file
View File

@ -0,0 +1,121 @@
package org.jetlinks.community.datasource.rdb.command;
import org.hswebframework.ezorm.rdb.executor.wrapper.MapResultWrapper;
import org.hswebframework.ezorm.rdb.operator.DMLOperator;
import org.hswebframework.ezorm.rdb.operator.DatabaseOperator;
import org.hswebframework.web.api.crud.entity.PagerResult;
import org.hswebframework.web.api.crud.entity.QueryParamEntity;
import org.hswebframework.web.crud.query.DefaultQueryHelper;
import org.hswebframework.web.crud.query.QueryHelper;
import org.jetlinks.core.command.CommandHandler;
import org.jetlinks.core.metadata.FunctionMetadata;
import org.jetlinks.core.metadata.PropertyMetadata;
import org.jetlinks.core.metadata.SimpleFunctionMetadata;
import org.jetlinks.core.metadata.SimplePropertyMetadata;
import org.jetlinks.core.metadata.types.StringType;
import org.jetlinks.community.datasource.DataSourceConstants;
import org.jetlinks.sdk.server.commons.cmd.QueryPagerCommand;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import java.util.*;
import java.util.function.Consumer;
import java.util.function.Function;
public class QueryPager extends QueryPagerCommand<Map<String, Object>>
implements RDBCommand<Mono<PagerResult<Map<String, Object>>>> {
private final static String PAGING_COLUMN = "ROWNUM_";
public String getTable() {
return String.valueOf(readable().get("table"));
}
public String getSql() {
return String.valueOf(readable().get("sql"));
}
public QueryParamEntity getParam() {
return this.asQueryParam();
}
@Override
public Mono<PagerResult<Map<String, Object>>> execute(DatabaseOperator operator) {
QueryParamEntity param = getParam();
String table = getTable();
String sql = getSql();
DMLOperator dmlOperator = operator.dml();
DefaultQueryHelper queryHelper = new DefaultQueryHelper(operator);
if (sql.isEmpty() || sql.equals("null")) {
return Mono.zip(
dmlOperator
.query(table)
.setParam(param)
.paging(param.getPageIndex(), param.getPageSize())
.fetch(new MapResultWrapper())
.reactive()
.map(map -> {
map.remove(PAGING_COLUMN);
return map;
})
.collectList(),
dmlOperator
.createReactiveRepository(table)
.createQuery()
.setParam(param)
.count(),
(list, total) -> PagerResult.of(total, list, param));
}
return QueryHelper
.transformPageResult(
queryHelper
.select(sql)
.where(param)
.fetchPaged(),
list -> Flux
.fromIterable(list)
.map(record -> (Map<String, Object>)record)
.collectList()
);
}
public static FunctionMetadata metadata() {
List<PropertyMetadata> list = getInputList();
return DataSourceConstants.Metadata
.create(QueryPager.class, func -> {
func.setName("分页查询");
func.setInputs(list);
});
}
public static List<PropertyMetadata> getInputList() {
List<PropertyMetadata> list = new ArrayList<>(QueryPagerCommand.getQueryParamMetadata());
list.add(SimplePropertyMetadata.of("table", "表名", StringType.GLOBAL));
list.add(SimplePropertyMetadata.of("sql", "sql语句", StringType.GLOBAL));
return list;
}
public static <T> CommandHandler<QueryPager, Mono<PagerResult<Map<String, Object>>>> createQueryHandler(
Consumer<SimpleFunctionMetadata> custom,
Function<QueryPager, Mono<PagerResult<Map<String, Object>>>> handler) {
return CommandHandler.of(
() -> metadata(custom),
(cmd, ignore) -> handler.apply(cmd),
QueryPager::new
);
}
public static FunctionMetadata metadata(Consumer<SimpleFunctionMetadata> custom) {
SimpleFunctionMetadata metadata = new SimpleFunctionMetadata();
metadata.setId("QueryPager");
metadata.setName("分页查询");
metadata.setInputs(getInputList());
custom.accept(metadata);
return metadata;
}
}

9
jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/RDBCommand.java Normal file
View File

@ -0,0 +1,9 @@
package org.jetlinks.community.datasource.rdb.command;
import org.hswebframework.ezorm.rdb.operator.DatabaseOperator;
import org.jetlinks.core.command.Command;
public interface RDBCommand<T> extends Command<T> {
T execute(DatabaseOperator operator);
}

65
jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/RDBRequestListCommand.java Normal file
View File

@ -0,0 +1,65 @@
package org.jetlinks.community.datasource.rdb.command;
import org.hswebframework.web.bean.FastBeanCopier;
import org.jetlinks.core.command.CommandHandler;
import org.jetlinks.core.metadata.FunctionMetadata;
import org.jetlinks.core.metadata.PropertyMetadata;
import org.jetlinks.core.metadata.SimpleFunctionMetadata;
import org.jetlinks.sdk.server.commons.cmd.QueryListCommand;
import reactor.core.publisher.Flux;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
import java.util.function.Function;
public class RDBRequestListCommand extends QueryListCommand<Map<String, Object>> {
private String commandId;
@Override
public String getCommandId() {
return commandId;
}
public RDBRequestListCommand(String commandId) {
this.commandId = commandId;
withConverter(RDBRequestListCommand::convertMap);
}
public RDBRequestListCommand() {
withConverter(RDBRequestListCommand::convertMap);
}
public static <T> CommandHandler<RDBRequestListCommand, Flux<Map<String, Object>>> createQueryHandler(
String commandId,
String commandName,
Consumer<SimpleFunctionMetadata> custom,
Function<RDBRequestListCommand, Flux<Map<String, Object>>> handler) {
return CommandHandler.of(
() -> metadata(commandId, commandName, custom),
(cmd, ignore) -> handler.apply(cmd),
() -> new RDBRequestListCommand(commandId)
);
}
public static FunctionMetadata metadata(String commandId, String commandName, Consumer<SimpleFunctionMetadata> custom) {
SimpleFunctionMetadata metadata = new SimpleFunctionMetadata();
List<PropertyMetadata> queryParamMetadata = QueryListCommand.getQueryParamMetadata();
metadata.setId(commandId);
metadata.setName(commandName);
metadata.setInputs(queryParamMetadata);
custom.accept(metadata);
return metadata;
}
public static Map<String, Object> convertMap(Object obj) {
return obj instanceof Map ? (Map<String, Object>) obj : FastBeanCopier.copy(obj, new HashMap<>());
}
}

58
jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/RDBRequestPagerCommand.java Normal file
View File

@ -0,0 +1,58 @@
package org.jetlinks.community.datasource.rdb.command;
import org.hswebframework.web.api.crud.entity.PagerResult;
import org.jetlinks.core.command.CommandHandler;
import org.jetlinks.core.metadata.FunctionMetadata;
import org.jetlinks.core.metadata.PropertyMetadata;
import org.jetlinks.core.metadata.SimpleFunctionMetadata;
import org.jetlinks.sdk.server.commons.cmd.QueryPagerCommand;
import reactor.core.publisher.Mono;
import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
import java.util.function.Function;
public class RDBRequestPagerCommand extends QueryPagerCommand<Map<String, Object>> {
private String commandId;
@Override
public String getCommandId() {
return commandId;
}
public RDBRequestPagerCommand(String commandId) {
this.commandId = commandId;
withConverter(RDBRequestListCommand::convertMap);
}
public RDBRequestPagerCommand() {
withConverter(RDBRequestListCommand::convertMap);
}
public static <T> CommandHandler<RDBRequestPagerCommand, Mono<PagerResult<Map<String, Object>>>> createQueryHandler(
String commandId,
String commandName,
Consumer<SimpleFunctionMetadata> custom,
Function<RDBRequestPagerCommand, Mono<PagerResult<Map<String, Object>>>> handler) {
return CommandHandler.of(
() -> metadata(commandId, commandName, custom),
(cmd, ignore) -> handler.apply(cmd),
() -> new RDBRequestPagerCommand(commandId)
);
}
public static FunctionMetadata metadata(String commandId, String commandName,Consumer<SimpleFunctionMetadata> custom) {
SimpleFunctionMetadata metadata = new SimpleFunctionMetadata();
List<PropertyMetadata> queryParamMetadata = QueryPagerCommand.getQueryParamMetadata();
metadata.setId(commandId);
metadata.setName(commandName);
metadata.setInputs(queryParamMetadata);
custom.accept(metadata);
return metadata;
}
}

30
jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/Refresh.java Normal file
View File

@ -0,0 +1,30 @@
package org.jetlinks.community.datasource.rdb.command;
import lombok.AllArgsConstructor;
import org.hswebframework.ezorm.rdb.metadata.RDBSchemaMetadata;
import org.hswebframework.ezorm.rdb.operator.DatabaseOperator;
import org.jetlinks.core.metadata.FunctionMetadata;
import org.jetlinks.community.datasource.DataSourceConstants;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
@AllArgsConstructor
public class Refresh implements RDBCommand<Mono<Void>> {
@Override
public Mono<Void> execute(DatabaseOperator operator) {
return Flux
.fromIterable(operator
.getMetadata()
.getSchemas())
.concatMap(RDBSchemaMetadata::loadAllTableReactive)
.then();
}
public static FunctionMetadata metadata() {
return DataSourceConstants.Metadata
.create(Refresh.class, func -> {
func.setName("刷新RDB数据源信息");
});
}
}

27
jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/Table.java Normal file
View File

@ -0,0 +1,27 @@
package org.jetlinks.community.datasource.rdb.command;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import org.hswebframework.ezorm.rdb.metadata.TableOrViewMetadata;
import java.util.List;
import java.util.stream.Collectors;
@Getter
@Setter
@AllArgsConstructor
@NoArgsConstructor
public class Table {
private String name;
private List<Column> columns;
public static Table of(TableOrViewMetadata metadata){
Table table = new Table();
table.setName(metadata.getName());
table.setColumns(metadata.getColumns().stream().map(Column::of).collect(Collectors.toList()));
return table;
}
}

39
jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/Upsert.java Normal file
View File

@ -0,0 +1,39 @@
package org.jetlinks.community.datasource.rdb.command;
import lombok.AllArgsConstructor;
import org.hswebframework.ezorm.rdb.operator.DatabaseOperator;
import reactor.core.publisher.Mono;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
@AllArgsConstructor
public class Upsert implements RDBCommand<Mono<Void>> {
private final String table;
private final List<Map<String, Object>> dataList;
private final Set<String> ignoreUpdateColumn;
public Upsert(String table, List<Map<String, Object>> dataList) {
this(table, dataList, Collections.emptySet());
}
@Override
public Mono<Void> execute(DatabaseOperator operator) {
return operator
.getMetadata()
.getTableReactive(table)
.flatMap(tableMetadata -> operator
.dml()
.upsert(tableMetadata)
.values(dataList)
.ignoreUpdate(ignoreUpdateColumn == null ? new String[0] : ignoreUpdateColumn.toArray(new String[0]))
.execute()
.reactive()
.then()
);
}
}

2
jetlinks-components/datasource-component/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports Normal file
View File

@ -0,0 +1,2 @@
org.jetlinks.community.datasource.configuration.DataSourceManagerConfiguration
org.jetlinks.community.datasource.configuration.DataSourceHandlerProviderConfiguration

3
jetlinks-components/datasource-component/src/main/resources/i18n/datasource/messages_en.properties Normal file
View File

@ -0,0 +1,3 @@
error.datasource_not_exist=The datasource {0}:{1} does not exist
error.unsupported_database_type=Unsupported database type
error.database_access_error=Database access error

3
jetlinks-components/datasource-component/src/main/resources/i18n/datasource/messages_zh.properties Normal file
View File

@ -0,0 +1,3 @@
error.datasource_not_exist=\u6570\u636E\u6E90{0}:{1}\u4E0D\u5B58\u5728
error.unsupported_database_type=\u4E0D\u652F\u6301\u7684\u6570\u636E\u5E93\u7C7B\u578B
error.database_access_error=\u6570\u636E\u5E93\u7528\u6237\u540D\u6216\u5BC6\u7801\u4E0D\u6B63\u786E

1
jetlinks-components/elasticsearch-component/.gitignore vendored Executable file
View File

@ -0,0 +1 @@
/data/

42
jetlinks-components/elasticsearch-component/elasticsearch-7x/pom.xml Normal file
View File

@ -0,0 +1,42 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.jetlinks.community</groupId>
<artifactId>elasticsearch-component</artifactId>
<version>2.10.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>elasticsearch-7x</artifactId>
<dependencies>
<dependency>
<groupId>org.jetlinks.community</groupId>
<artifactId>elasticsearch-core</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>co.elastic.clients</groupId>
<artifactId>elasticsearch-java</artifactId>
<version>${elasticsearch7x.version}</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-client</artifactId>
<version>${elasticsearch7x.version}</version>
</dependency>
<dependency>
<groupId>org.jetlinks.community</groupId>
<artifactId>things-component</artifactId>
<optional>true</optional>
</dependency>
</dependencies>
</project>

67
jetlinks-components/elasticsearch-component/elasticsearch-7x/src/main/java/org/jetlinks/community/elastic/search/ElasticSearch7xSupport.java Normal file
View File

@ -0,0 +1,67 @@
package org.jetlinks.community.elastic.search;
import co.elastic.clients.elasticsearch._types.aggregations.DateHistogramBucket;
import co.elastic.clients.elasticsearch._types.aggregations.HistogramBucket;
import co.elastic.clients.elasticsearch._types.aggregations.MultiBucketBase;
import co.elastic.clients.elasticsearch._types.mapping.DynamicTemplate;
import co.elastic.clients.elasticsearch._types.mapping.Property;
import co.elastic.clients.elasticsearch.indices.*;
import co.elastic.clients.elasticsearch.indices.get_mapping.IndexMappingRecord;
import co.elastic.clients.transport.Version;
import org.jetlinks.community.elastic.search.enums.ElasticSearchTermTypes;
import org.jetlinks.community.elastic.search.enums.ElasticSearch7xTermType;
import org.jetlinks.community.elastic.search.index.ElasticSearchIndexProperties;
public class ElasticSearch7xSupport extends ElasticSearchSupport {
static {
Version version = Version.VERSION;
if (version != null && version.major() == 7) {
for (ElasticSearch7xTermType value : ElasticSearch7xTermType.values()) {
ElasticSearchTermTypes.register(value);
}
}
}
@Override
public IndexSettings.Builder applyIndexSettings(ElasticSearchIndexProperties index, IndexSettings.Builder builder) {
return super
.applyIndexSettings(index, builder)
.mapping(b -> b
.totalFields(t -> t.limit((int) index.getTotalFieldsLimit())));
}
@Override
public DynamicTemplate createDynamicTemplate(String type, Property property) {
return DynamicTemplate
.of(b -> b
.matchMappingType(type)
.mapping(property));
}
@Override
public TemplateMapping getTemplateMapping(GetTemplateResponse response, String index) {
return response.get(index);
}
@Override
public IndexState getIndexState(GetIndexResponse response, String index) {
return response.get(index);
}
@Override
public IndexMappingRecord getIndexMapping(GetMappingResponse response, String index) {
return response.get(index);
}
@Override
protected Object getBucketKey(MultiBucketBase bucket) {
if (bucket instanceof DateHistogramBucket _bucket) {
return _bucket.key();
}
if (bucket instanceof HistogramBucket _bucket) {
return _bucket.key();
}
return null;
}
}

232
jetlinks-components/elasticsearch-component/elasticsearch-7x/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticSearch7xTermType.java Executable file
View File

@ -0,0 +1,232 @@
package org.jetlinks.community.elastic.search.enums;
import co.elastic.clients.elasticsearch._types.FieldValue;
import co.elastic.clients.elasticsearch._types.query_dsl.Query;
import co.elastic.clients.json.JsonData;
import lombok.AllArgsConstructor;
import lombok.Getter;
import org.hswebframework.ezorm.core.param.Term;
import org.hswebframework.ezorm.core.param.TermType;
import org.jetlinks.core.metadata.DataType;
import org.jetlinks.community.utils.ConverterUtils;
import org.jetlinks.reactor.ql.utils.CastUtils;
import org.springframework.util.ObjectUtils;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Optional;
/**
* @author Jia_RG , bestfeng ,zhouhao
*/
@Getter
@AllArgsConstructor
public enum ElasticSearch7xTermType implements ElasticSearchTermType {
eq(TermType.eq) {
@Override
public Query.Builder process(Term term, Query.Builder builder) {
builder.term(termQuery -> termQuery
.field(term.getColumn().trim())
.value(FieldValue.of(JsonData.of(term.getValue()))));
return builder;
}
},
not(TermType.not) {
@Override
public Query.Builder process(Term term, Query.Builder builder) {
builder.bool(bool -> bool
.mustNot(mustNot -> mustNot
.term(termQuery -> termQuery
.field(term.getColumn().trim())
.value(FieldValue.of(JsonData.of(term.getValue()))))));
return builder;
}
},
isnull(TermType.isnull) {
@Override
public Query.Builder process(Term term, Query.Builder builder) {
builder
.bool(bool -> bool
.mustNot(mustNot -> notnull.process(term, mustNot)));
return builder;
}
@Override
public Object convertTermValue(DataType type, Object value) {
return value;
}
},
notnull(TermType.notnull) {
@Override
public Query.Builder process(Term term, Query.Builder builder) {
builder.exists(exists -> exists.field(term.getColumn()));
return builder;
}
@Override
public Object convertTermValue(DataType type, Object value) {
return value;
}
},
empty(TermType.empty) {
@Override
public Query.Builder process(Term term, Query.Builder builder) {
builder.term(t -> t.field(term.getColumn().trim()).value(""));
return builder;
}
@Override
public Object convertTermValue(DataType type, Object value) {
return value;
}
},
nempty(TermType.nempty) {
@Override
public Query.Builder process(Term term, Query.Builder builder) {
builder.bool(bool -> bool
.mustNot(mustNot -> empty.process(term, mustNot)));
return builder;
}
@Override
public Object convertTermValue(DataType type, Object value) {
return value;
}
},
btw(TermType.btw) {
@Override
public Query.Builder process(Term term, Query.Builder builder) {
Number between = null;
Number and = null;
List<?> values = ConverterUtils.convertToList(term.getValue());
int size = values.size();
if (size > 0) {
between = CastUtils.castNumber(values.get(0));
}
if (size > 1) {
and = CastUtils.castNumber(values.get(1));
}
Number fb = between, ab = and;
builder.range(range -> range
.field(term.getColumn())
.gte(JsonData.of(fb))
.lte(JsonData.of(ab)));
return builder;
}
},
gt(TermType.gt) {
@Override
public Query.Builder process(Term term, Query.Builder builder) {
Number value = CastUtils.castNumber(term.getValue());
builder.range(range -> range
.field(term.getColumn())
.gt(JsonData.of(value)));
return builder;
}
},
gte(TermType.gte) {
@Override
public Query.Builder process(Term term, Query.Builder builder) {
Number value = CastUtils.castNumber(term.getValue());
builder.range(range -> range
.field(term.getColumn())
.gte(JsonData.of(value)));
return builder;
}
},
lt(TermType.lt) {
@Override
public Query.Builder process(Term term, Query.Builder builder) {
Number value = CastUtils.castNumber(term.getValue());
builder.range(range -> range
.field(term.getColumn())
.lt(JsonData.of(value)));
return builder;
}
},
lte(TermType.lte) {
@Override
public Query.Builder process(Term term, Query.Builder builder) {
Number value = CastUtils.castNumber(term.getValue());
builder.range(range -> range
.field(term.getColumn())
.lte(JsonData.of(value)));
return builder;
}
},
in(TermType.in) {
@Override
public Query.Builder process(Term term, Query.Builder builder) {
builder.terms(termQuery -> termQuery
.field(term.getColumn().trim())
.terms(t -> t.value(
ConverterUtils.convertToList(term.getValue(), d->FieldValue.of(JsonData.of(d)))
)));
return builder;
}
},
nin(TermType.nin) {
@Override
public Query.Builder process(Term term, Query.Builder builder) {
builder.bool(bool -> bool
.mustNot(mustNot -> in.process(term, mustNot)));
return builder;
}
},
like(TermType.like) {
@Override
public Query.Builder process(Term term, Query.Builder builder) {
builder.wildcard(wildcard -> wildcard
.field(term.getColumn().trim())
.value(likeQueryTermValueHandler(term.getValue())));
return builder;
}
},
nlike(TermType.nlike) {
@Override
public Query.Builder process(Term term, Query.Builder builder) {
builder.bool(bool -> bool
.mustNot(mustNot -> mustNot
.wildcard(wildcard -> wildcard
.field(term.getColumn().trim())
.value(likeQueryTermValueHandler(term.getValue())))));
return builder;
}
};
private final String type;
@Override
public boolean isSupported(Term term) {
return this.type.equalsIgnoreCase(term.getTermType());
}
@Override
public String getId() {
return type;
}
private static String convertString(Object value) {
if (value instanceof Collection) {
return String.join(",", ((Collection<String>) value));
} else {
return String.valueOf(value);
}
}
public static String likeQueryTermValueHandler(Object value) {
if (!ObjectUtils.isEmpty(value)) {
return convertString(value).replace("%", "*");
}
return "**";
}
public static Optional<ElasticSearch7xTermType> of(String type) {
return Arrays.stream(values())
.filter(e -> e.getType().equalsIgnoreCase(type))
.findAny();
}
}

1
jetlinks-components/elasticsearch-component/elasticsearch-7x/src/main/resources/META-INF/services/org.jetlinks.community.elastic.search.ElasticSearchSupport Normal file
View File

@ -0,0 +1 @@
org.jetlinks.community.elastic.search.ElasticSearch7xSupport

40
jetlinks-components/elasticsearch-component/elasticsearch-8x/pom.xml Normal file
View File

@ -0,0 +1,40 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.jetlinks.community</groupId>
<artifactId>elasticsearch-component</artifactId>
<version>2.10.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>elasticsearch-8x</artifactId>
<dependencies>
<dependency>
<groupId>org.jetlinks.community</groupId>
<artifactId>elasticsearch-core</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>co.elastic.clients</groupId>
<artifactId>elasticsearch-java</artifactId>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-client</artifactId>
</dependency>
<dependency>
<groupId>org.jetlinks.community</groupId>
<artifactId>things-component</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project>

73
jetlinks-components/elasticsearch-component/elasticsearch-8x/src/main/java/org/jetlinks/community/elastic/search/ElasticSearch8xSupport.java Normal file
View File

@ -0,0 +1,73 @@
package org.jetlinks.community.elastic.search;
import co.elastic.clients.elasticsearch._types.aggregations.DateHistogramBucket;
import co.elastic.clients.elasticsearch._types.aggregations.HistogramBucket;
import co.elastic.clients.elasticsearch._types.aggregations.MultiBucketBase;
import co.elastic.clients.elasticsearch._types.mapping.DynamicTemplate;
import co.elastic.clients.elasticsearch._types.mapping.Property;
import co.elastic.clients.elasticsearch.indices.*;
import co.elastic.clients.elasticsearch.indices.get_mapping.IndexMappingRecord;
import co.elastic.clients.transport.Version;
import org.jetlinks.community.elastic.search.enums.ElasticSearchTermTypes;
import org.jetlinks.community.elastic.search.enums.ElasticSearch8xTermType;
import org.jetlinks.community.elastic.search.index.ElasticSearchIndexProperties;
public class ElasticSearch8xSupport extends ElasticSearchSupport {
static {
Version version = Version.VERSION;
if (version != null && version.major() == 8) {
for (ElasticSearch8xTermType value : ElasticSearch8xTermType.values()) {
ElasticSearchTermTypes.register(value);
}
}
}
@Override
public boolean is8x() {
return true;
}
@Override
public IndexSettings.Builder applyIndexSettings(ElasticSearchIndexProperties index, IndexSettings.Builder builder) {
return super
.applyIndexSettings(index, builder)
.mapping(b -> b
.totalFields(t -> t.limit(index.getTotalFieldsLimit())));
}
@Override
public DynamicTemplate createDynamicTemplate(String type, Property property) {
return DynamicTemplate
.of(b -> b
.matchMappingType(type)
.mapping(property));
}
@Override
public TemplateMapping getTemplateMapping(GetTemplateResponse response, String index) {
return response.get(index);
}
@Override
public IndexState getIndexState(GetIndexResponse response, String index) {
return response.get(index);
}
@Override
public IndexMappingRecord getIndexMapping(GetMappingResponse response, String index) {
return response.get(index);
}
@Override
protected Object getBucketKey(MultiBucketBase bucket) {
if (bucket instanceof DateHistogramBucket _bucket) {
return _bucket.key();
}
if (bucket instanceof HistogramBucket _bucket) {
return _bucket.key();
}
return null;
}
}

238
jetlinks-components/elasticsearch-component/elasticsearch-8x/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticSearch8xTermType.java Executable file
View File

@ -0,0 +1,238 @@
package org.jetlinks.community.elastic.search.enums;
import co.elastic.clients.elasticsearch._types.FieldValue;
import co.elastic.clients.elasticsearch._types.query_dsl.Query;
import co.elastic.clients.json.JsonData;
import lombok.AllArgsConstructor;
import lombok.Getter;
import org.hswebframework.ezorm.core.param.Term;
import org.hswebframework.ezorm.core.param.TermType;
import org.jetlinks.core.metadata.DataType;
import org.jetlinks.community.utils.ConverterUtils;
import org.jetlinks.reactor.ql.utils.CastUtils;
import org.springframework.util.ObjectUtils;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Optional;
/**
* @author Jia_RG , bestfeng ,zhouhao
*/
@Getter
@AllArgsConstructor
public enum ElasticSearch8xTermType implements ElasticSearchTermType {
eq(TermType.eq) {
@Override
public Query.Builder process(Term term, Query.Builder builder) {
builder.term(termQuery -> termQuery
.field(term.getColumn().trim())
.value(FieldValue.of(term.getValue())));
return builder;
}
},
not(TermType.not) {
@Override
public Query.Builder process(Term term, Query.Builder builder) {
builder.bool(bool -> bool
.mustNot(mustNot -> mustNot
.term(termQuery -> termQuery
.field(term.getColumn().trim())
.value(FieldValue.of(term.getValue())))));
return builder;
}
},
isnull(TermType.isnull) {
@Override
public Query.Builder process(Term term, Query.Builder builder) {
builder
.bool(bool -> bool
.mustNot(mustNot -> notnull.process(term, mustNot)));
return builder;
}
@Override
public Object convertTermValue(DataType type, Object value) {
return value;
}
},
notnull(TermType.notnull) {
@Override
public Query.Builder process(Term term, Query.Builder builder) {
builder.exists(exists -> exists.field(term.getColumn()));
return builder;
}
@Override
public Object convertTermValue(DataType type, Object value) {
return value;
}
},
empty(TermType.empty) {
@Override
public Query.Builder process(Term term, Query.Builder builder) {
builder.term(t -> t.field(term.getColumn().trim()).value(""));
return builder;
}
@Override
public Object convertTermValue(DataType type, Object value) {
return value;
}
},
nempty(TermType.nempty) {
@Override
public Query.Builder process(Term term, Query.Builder builder) {
builder.bool(bool -> bool
.mustNot(mustNot -> empty.process(term, mustNot)));
return builder;
}
@Override
public Object convertTermValue(DataType type, Object value) {
return value;
}
},
btw(TermType.btw) {
@Override
public Query.Builder process(Term term, Query.Builder builder) {
Number between = null;
Number and = null;
List<?> values = ConverterUtils.convertToList(term.getValue());
int size = values.size();
if (size > 0) {
between = CastUtils.castNumber(values.get(0));
}
if (size > 1) {
and = CastUtils.castNumber(values.get(1));
}
Number fb = between, ab = and;
builder.range(range -> range
.untyped(numberRange -> {
numberRange.gte(JsonData.of(fb));
numberRange.lte(JsonData.of(ab));
return numberRange.field(term.getColumn());
}));
return builder;
}
},
gt(TermType.gt) {
@Override
public Query.Builder process(Term term, Query.Builder builder) {
Number value = CastUtils.castNumber(term.getValue());
builder.range(range -> range
.untyped(untyped -> untyped
.field(term.getColumn())
.gt(JsonData.of(value))));
return builder;
}
},
gte(TermType.gte) {
@Override
public Query.Builder process(Term term, Query.Builder builder) {
Number value = CastUtils.castNumber(term.getValue());
builder.range(range -> range
.untyped(untyped -> untyped
.field(term.getColumn())
.gte(JsonData.of(value))));
return builder;
}
},
lt(TermType.lt) {
@Override
public Query.Builder process(Term term, Query.Builder builder) {
Number value = CastUtils.castNumber(term.getValue());
builder.range(range -> range
.untyped(untyped -> untyped
.field(term.getColumn())
.lt(JsonData.of(value))));
return builder;
}
},
lte(TermType.lte) {
@Override
public Query.Builder process(Term term, Query.Builder builder) {
Number value = CastUtils.castNumber(term.getValue());
builder.range(range -> range
.untyped(untyped -> untyped
.field(term.getColumn())
.lte(JsonData.of(value))));
return builder;
}
},
in(TermType.in) {
@Override
public Query.Builder process(Term term, Query.Builder builder) {
builder.terms(termQuery -> termQuery
.field(term.getColumn().trim())
.terms(t -> t.value(
ConverterUtils.convertToList(term.getValue(), FieldValue::of)
)));
return builder;
}
},
nin(TermType.nin) {
@Override
public Query.Builder process(Term term, Query.Builder builder) {
builder.bool(bool -> bool
.mustNot(mustNot -> in.process(term, mustNot)));
return builder;
}
},
like(TermType.like) {
@Override
public Query.Builder process(Term term, Query.Builder builder) {
builder.wildcard(wildcard -> wildcard
.field(term.getColumn().trim())
.value(likeQueryTermValueHandler(term.getValue())));
return builder;
}
},
nlike(TermType.nlike) {
@Override
public Query.Builder process(Term term, Query.Builder builder) {
builder.bool(bool -> bool
.mustNot(mustNot -> mustNot
.wildcard(wildcard -> wildcard
.field(term.getColumn().trim())
.value(likeQueryTermValueHandler(term.getValue())))));
return builder;
}
};
private final String type;
@Override
public boolean isSupported(Term term) {
return this.type.equalsIgnoreCase(term.getTermType());
}
@Override
public String getId() {
return name();
}
private static String convertString(Object value) {
if (value instanceof Collection) {
return String.join(",", ((Collection<String>) value));
} else {
return String.valueOf(value);
}
}
public static String likeQueryTermValueHandler(Object value) {
if (!ObjectUtils.isEmpty(value)) {
return convertString(value).replace("%", "*");
}
return "**";
}
public static Optional<ElasticSearch8xTermType> of(String type) {
return Arrays.stream(values())
.filter(e -> e.getType().equalsIgnoreCase(type))
.findAny();
}
}

1
jetlinks-components/elasticsearch-component/elasticsearch-8x/src/main/resources/META-INF/services/org.jetlinks.community.elastic.search.ElasticSearchSupport Normal file
View File

@ -0,0 +1 @@
org.jetlinks.community.elastic.search.ElasticSearch8xSupport

129
jetlinks-components/elasticsearch-component/elasticsearch-core/pom.xml Normal file
View File

@ -0,0 +1,129 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.jetlinks.community</groupId>
<artifactId>elasticsearch-component</artifactId>
<version>2.10.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>elasticsearch-core</artifactId>
<dependencies>
<!-- <dependency>&lt;!&ndash; required by elasticsearch &ndash;&gt;-->
<!-- <groupId>org.elasticsearch.plugin</groupId>-->
<!-- <artifactId>transport-netty4-client</artifactId>-->
<!-- <exclusions>-->
<!-- <exclusion>-->
<!-- <groupId>*</groupId>-->
<!-- <artifactId>*</artifactId>-->
<!-- </exclusion>-->
<!-- </exclusions>-->
<!-- </dependency>-->
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>log4j-over-slf4j</artifactId>
</dependency>
<dependency>
<groupId>org.locationtech.jts</groupId>
<artifactId>jts-core</artifactId>
<version>1.18.2</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.locationtech.spatial4j/spatial4j -->
<dependency>
<groupId>org.locationtech.spatial4j</groupId>
<artifactId>spatial4j</artifactId>
<version>0.8</version>
</dependency>
<!-- <dependency>-->
<!-- <groupId>org.elasticsearch.client</groupId>-->
<!-- <artifactId>elasticsearch-rest-high-level-client</artifactId>-->
<!-- </dependency>-->
<!-- <dependency>-->
<!-- <groupId>org.springframework.data</groupId>-->
<!-- <artifactId>spring-data-elasticsearch</artifactId>-->
<!-- </dependency>-->
<dependency>
<groupId>org.hswebframework.web</groupId>
<artifactId>hsweb-commons-crud</artifactId>
<version>${hsweb.framework.version}</version>
</dependency>
<dependency>
<groupId>org.hswebframework</groupId>
<artifactId>hsweb-easy-orm-rdb</artifactId>
</dependency>
<dependency>
<groupId>org.jetlinks</groupId>
<artifactId>jetlinks-core</artifactId>
<version>${jetlinks.version}</version>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>timeseries-component</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>io.projectreactor.netty</groupId>
<artifactId>reactor-netty</artifactId>
</dependency>
<dependency>
<groupId>org.jetlinks.community</groupId>
<artifactId>configure-component</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.jetlinks.community</groupId>
<artifactId>things-component</artifactId>
<version>${project.version}</version>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.springframework.data</groupId>
<artifactId>spring-data-elasticsearch</artifactId>
<exclusions>
<exclusion>
<groupId>co.elastic.clients</groupId>
<artifactId>elasticsearch-java</artifactId>
</exclusion>
<exclusion>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-client</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>co.elastic.clients</groupId>
<artifactId>elasticsearch-java</artifactId>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-client</artifactId>
</dependency>
</dependencies>
</project>

111
jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/ElasticSearchSupport.java Normal file
View File

@ -0,0 +1,111 @@
package org.jetlinks.community.elastic.search;
import co.elastic.clients.elasticsearch._types.aggregations.DateHistogramBucket;
import co.elastic.clients.elasticsearch._types.aggregations.HistogramBucket;
import co.elastic.clients.elasticsearch._types.aggregations.MultiBucketBase;
import co.elastic.clients.elasticsearch._types.mapping.DynamicTemplate;
import co.elastic.clients.elasticsearch._types.mapping.Property;
import co.elastic.clients.elasticsearch.indices.*;
import co.elastic.clients.elasticsearch.indices.get_mapping.IndexMappingRecord;
import co.elastic.clients.json.JsonData;
import co.elastic.clients.transport.Version;
import com.google.common.collect.Maps;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.MapUtils;
import org.jetlinks.community.elastic.search.index.ElasticSearchIndexProperties;
import java.util.HashMap;
import java.util.Map;
import java.util.ServiceLoader;
@Slf4j
public abstract class ElasticSearchSupport {
private static ElasticSearchSupport support;
static {
ServiceLoader
.load(ElasticSearchSupport.class)
.forEach(ElasticSearchSupport::register);
}
public static void load() {
}
public int majorVersion() {
return Version.VERSION == null ? 8 : Version.VERSION.major();
}
public boolean is8x() {
return majorVersion() == 8;
}
public boolean is7x() {
return majorVersion() == 7;
}
public static ElasticSearchSupport current() {
if (support == null) {
throw new UnsupportedOperationException("当前环境不支持elasticsearch,请添加依赖`elasticsearch-8x`.");
}
return support;
}
private static void register(ElasticSearchSupport support) {
if (ElasticSearchSupport.support != null) {
log.warn("ignore register elasticsearch support:{}", support.getClass());
return;
}
support.setup();
ElasticSearchSupport.support = support;
log.info("register elasticsearch support:{}", support.getClass());
}
protected void setup() {
}
public IndexSettings.Builder applyIndexSettings(ElasticSearchIndexProperties index,
IndexSettings.Builder builder) {
builder.numberOfShards(String.valueOf(Math.max(1, index.getNumberOfShards())))
.numberOfReplicas(String.valueOf(index.getNumberOfReplicas()));
if (MapUtils.isNotEmpty(index.getOptions())) {
builder.otherSettings(Maps.transformValues(index.getOptions(), JsonData::of));
}
return builder;
}
public abstract DynamicTemplate createDynamicTemplate(String type, Property property);
public abstract IndexState getIndexState(GetIndexResponse response, String index);
public abstract TemplateMapping getTemplateMapping(GetTemplateResponse response, String index);
public abstract IndexMappingRecord getIndexMapping(GetMappingResponse response, String index);
protected abstract Object getBucketKey(MultiBucketBase bucket);
public Map<String, Object> transformBucket(String name,
Map<String, Object> map,
MultiBucketBase bucket) {
if (bucket instanceof DateHistogramBucket _bucket) {
Map<String, Object> val = new HashMap<>(map);
val.put(name, _bucket.keyAsString());
val.put("_" + name, getBucketKey(_bucket));
return val;
} else if (bucket instanceof HistogramBucket _bucket) {
Map<String, Object> val = new HashMap<>(map);
val.put(name, _bucket.keyAsString());
val.put("_" + name, getBucketKey(_bucket));
return val;
}
return map;
}
}

44
jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/configuration/ElasticSearchClientConfiguration.java Executable file
View File

@ -0,0 +1,44 @@
package org.jetlinks.community.elastic.search.configuration;
import co.elastic.clients.json.JsonpMapper;
import co.elastic.clients.json.jackson.JacksonJsonpMapper;
import co.elastic.clients.transport.rest_client.RestClientOptions;
import co.elastic.clients.transport.rest_client.RestClientTransport;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.client.RestClient;
import org.jetlinks.community.elastic.search.trace.TraceInstrumentation;
import org.springframework.beans.factory.ObjectProvider;
import org.springframework.beans.factory.config.BeanDefinition;
import org.springframework.boot.autoconfigure.AutoConfiguration;
import org.springframework.boot.autoconfigure.elasticsearch.ReactiveElasticsearchClientAutoConfiguration;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Role;
/**
* @author zhouhao
* @since 2.10
**/
@AutoConfiguration(before = ReactiveElasticsearchClientAutoConfiguration.class)
@Slf4j
@Role(BeanDefinition.ROLE_INFRASTRUCTURE)
public class ElasticSearchClientConfiguration {
@Bean
JacksonJsonpMapper jacksonJsonpMapper(ObjectMapper mapper) {
return new JacksonJsonpMapper(mapper);
}
@Bean
RestClientTransport restClientTransport(RestClient restClient, JsonpMapper jsonMapper,
ObjectProvider<RestClientOptions> restClientOptions) {
return new RestClientTransport(
restClient,
jsonMapper,
restClientOptions.getIfAvailable(),
new TraceInstrumentation());
}
}

101
jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/configuration/ElasticSearchConfiguration.java → jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/configuration/ElasticSearchConfiguration.java
View File

@ -1,35 +1,38 @@
package org.jetlinks.community.elastic.search.configuration;
import co.elastic.clients.elasticsearch.ElasticsearchClient;
import co.elastic.clients.json.JsonpMapper;
import co.elastic.clients.json.jackson.JacksonJsonpMapper;
import co.elastic.clients.transport.rest_client.RestClientOptions;
import co.elastic.clients.transport.rest_client.RestClientTransport;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.Generated;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.jetlinks.community.elastic.search.embedded.EmbeddedElasticSearch;
import org.jetlinks.community.elastic.search.embedded.EmbeddedElasticSearchProperties;
import org.jetlinks.community.elastic.search.index.DefaultElasticSearchIndexManager;
import org.jetlinks.community.elastic.search.index.ElasticSearchIndexManager;
import org.jetlinks.community.elastic.search.index.ElasticSearchIndexProperties;
import org.jetlinks.community.elastic.search.index.ElasticSearchIndexStrategy;
import org.jetlinks.community.elastic.search.index.strategies.DirectElasticSearchIndexStrategy;
import org.jetlinks.community.elastic.search.index.strategies.TimeByMonthElasticSearchIndexStrategy;
import org.elasticsearch.client.RestClient;
import org.jetlinks.community.elastic.search.index.*;
import org.jetlinks.community.elastic.search.index.strategies.*;
import org.jetlinks.community.elastic.search.service.reactive.*;
import org.jetlinks.community.elastic.search.index.*;
import org.jetlinks.community.elastic.search.index.strategies.*;
import org.jetlinks.community.elastic.search.service.AggregationService;
import org.jetlinks.community.elastic.search.service.ElasticSearchService;
import org.jetlinks.community.elastic.search.service.reactive.*;
import org.jetlinks.community.elastic.search.timeseries.ElasticSearchTimeSeriesManager;
import org.springframework.beans.factory.ObjectProvider;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.AutoConfigureAfter;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
import org.springframework.beans.factory.config.BeanDefinition;
import org.springframework.boot.autoconfigure.AutoConfiguration;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.autoconfigure.data.elasticsearch.ReactiveElasticsearchRestClientAutoConfiguration;
import org.springframework.boot.autoconfigure.elasticsearch.ReactiveElasticsearchClientAutoConfiguration;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.DependsOn;
import org.springframework.context.annotation.Primary;
import org.springframework.data.elasticsearch.client.ClientConfiguration;
import org.springframework.data.elasticsearch.client.reactive.HostProvider;
import org.springframework.data.elasticsearch.client.reactive.RequestCreator;
import org.springframework.data.elasticsearch.client.reactive.WebClientProvider;
import org.springframework.context.annotation.Role;
import org.springframework.core.Ordered;
import org.springframework.core.annotation.Order;
import java.net.InetSocketAddress;
import java.util.List;
/**
@ -37,50 +40,24 @@ import java.util.List;
* @author zhouhao
* @since 1.0
**/
@Configuration(proxyBeanMethods = false)
@AutoConfiguration(after = ReactiveElasticsearchClientAutoConfiguration.class)
@Slf4j
@EnableConfigurationProperties({
EmbeddedElasticSearchProperties.class,
ElasticSearchIndexProperties.class,
ElasticSearchBufferProperties.class})
@AutoConfigureAfter(ReactiveElasticsearchRestClientAutoConfiguration.class)
@ConditionalOnBean(ClientConfiguration.class)
@Generated
@Role(BeanDefinition.ROLE_INFRASTRUCTURE)
public class ElasticSearchConfiguration {
@Bean
@SneakyThrows
@Primary
public DefaultReactiveElasticsearchClient defaultReactiveElasticsearchClient(EmbeddedElasticSearchProperties embeddedProperties,
ClientConfiguration clientConfiguration) {
if (embeddedProperties.isEnabled()) {
log.debug("starting embedded elasticsearch on {}:{}",
embeddedProperties.getHost(),
embeddedProperties.getPort());
new EmbeddedElasticSearch(embeddedProperties).start();
}
WebClientProvider provider = getWebClientProvider(clientConfiguration);
HostProvider<?> hostProvider = HostProvider.provider(provider,
clientConfiguration.getHeadersSupplier(),
clientConfiguration
.getEndpoints()
.toArray(new InetSocketAddress[0]));
DefaultReactiveElasticsearchClient client =
new DefaultReactiveElasticsearchClient(hostProvider, new RequestCreator() {
});
client.setHeadersSupplier(clientConfiguration.getHeadersSupplier());
return client;
@Role(BeanDefinition.ROLE_INFRASTRUCTURE)
public DefaultReactiveElasticsearchClient defaultReactiveElasticsearchClient(ElasticsearchClient client) {
return new DefaultReactiveElasticsearchClient(client);
}
private static WebClientProvider getWebClientProvider(ClientConfiguration clientConfiguration) {
return WebClientProvider.getWebClientProvider(clientConfiguration);
}
@Bean
public DirectElasticSearchIndexStrategy directElasticSearchIndexStrategy(ReactiveElasticsearchClient elasticsearchClient,
@ -89,18 +66,40 @@ public class ElasticSearchConfiguration {
}
@Bean
@Role(BeanDefinition.ROLE_INFRASTRUCTURE)
public TimeByMonthElasticSearchIndexStrategy timeByMonthElasticSearchIndexStrategy(ReactiveElasticsearchClient elasticsearchClient,
ElasticSearchIndexProperties indexProperties) {
return new TimeByMonthElasticSearchIndexStrategy(elasticsearchClient, indexProperties);
}
@Bean
public DefaultElasticSearchIndexManager elasticSearchIndexManager(@Autowired(required = false) List<ElasticSearchIndexStrategy> strategies) {
return new DefaultElasticSearchIndexManager(strategies);
public TimeByDayElasticSearchIndexStrategy timeByDayElasticSearchIndexStrategy(ReactiveElasticsearchClient elasticsearchClient,
ElasticSearchIndexProperties indexProperties) {
return new TimeByDayElasticSearchIndexStrategy(elasticsearchClient, indexProperties);
}
@Bean
public TimeByWeekElasticSearchIndexStrategy timeByWeekElasticSearchIndexStrategy(ReactiveElasticsearchClient elasticsearchClient,
ElasticSearchIndexProperties indexProperties){
return new TimeByWeekElasticSearchIndexStrategy(elasticsearchClient, indexProperties);
}
@Bean
public AffixesElasticSearchIndexStrategy affixesElasticSearchIndexStrategy(ReactiveElasticsearchClient elasticsearchClient,
ElasticSearchIndexProperties indexProperties) {
return new AffixesElasticSearchIndexStrategy(elasticsearchClient, indexProperties);
}
@Bean
public DefaultElasticSearchIndexManager elasticSearchIndexManager(@Autowired(required = false) List<ElasticSearchIndexStrategy> strategies,
@Autowired(required = false) List<ElasticSearchIndexCustomizer> customizers) {
return new DefaultElasticSearchIndexManager(strategies, customizers);
}
@Order(Ordered.HIGHEST_PRECEDENCE + 1000)
@Bean(destroyMethod = "shutdown")
@ConditionalOnMissingBean(ElasticSearchService.class)
@DependsOn("defaultReactiveElasticsearchClient")
public ReactiveElasticSearchService reactiveElasticSearchService(ReactiveElasticsearchClient elasticsearchClient,
ElasticSearchIndexManager indexManager,
ElasticSearchBufferProperties properties) {

6
jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/configuration/ElasticSearchProperties.java Normal file
View File

@ -0,0 +1,6 @@
package org.jetlinks.community.elastic.search.configuration;
public class ElasticSearchProperties {
}

3
jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/configuration/ElasticSearchThingDataConfiguration.java → jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/configuration/ElasticSearchThingDataConfiguration.java
View File

@ -7,6 +7,7 @@ import org.jetlinks.community.elastic.search.service.ElasticSearchService;
import org.jetlinks.community.elastic.search.things.ElasticSearchColumnModeStrategy;
import org.jetlinks.community.elastic.search.things.ElasticSearchRowModeStrategy;
import org.jetlinks.community.things.data.ThingsDataRepositoryStrategy;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@ -16,6 +17,7 @@ import org.springframework.context.annotation.Configuration;
public class ElasticSearchThingDataConfiguration {
@Bean
@ConditionalOnBean(ElasticSearchService.class)
public ElasticSearchColumnModeStrategy elasticSearchColumnModThingDataPolicy(
ThingsRegistry registry,
ElasticSearchService searchService,
@ -26,6 +28,7 @@ public class ElasticSearchThingDataConfiguration {
}
@Bean
@ConditionalOnBean(ElasticSearchService.class)
public ElasticSearchRowModeStrategy elasticSearchRowModThingDataPolicy(
ThingsRegistry registry,
ElasticSearchService searchService,

13
jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticDateFormat.java → jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticDateFormat.java Normal file → Executable file
View File

@ -9,9 +9,10 @@ import java.util.List;
import java.util.stream.Collectors;
/**
* Values based on reference doc - https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-date-format.html
*
* @author bsetfeng
* @since 1.0
* Values based on reference doc - https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-date-format.html
**/
@Getter
@AllArgsConstructor
@ -24,11 +25,11 @@ public enum ElasticDateFormat implements EnumDict<String> {
strict_date_time("strict_date_time", "yyyy-MM-dd'T'HH:mm:ss.SSSZZ"),
strict_date_hour_minute_second("strict_date_hour_minute_second", "yyyy-MM-dd'T'HH:mm:ss"),
strict_hour_minute_second("strict_hour_minute_second", "HH:mm:ss"),
simple_date("yyyy-MM-dd HH:mm:ss", "通用格式");
simple_date("8yyyy-MM-dd HH:mm:ss", "通用格式");
private String value;
private final String text;
private String text;
public static String getFormat(ElasticDateFormat... dateFormats) {
return getFormat(Arrays.asList(dateFormats));
@ -36,13 +37,13 @@ public enum ElasticDateFormat implements EnumDict<String> {
public static String getFormat(List<ElasticDateFormat> dateFormats) {
return getFormatStr(dateFormats.stream()
.map(ElasticDateFormat::getValue)
.collect(Collectors.toList())
.map(ElasticDateFormat::getValue)
.collect(Collectors.toList())
);
}
public static String getFormatStr(List<String> dateFormats) {
StringBuffer format = new StringBuffer();
StringBuilder format = new StringBuilder();
for (int i = 0; i < dateFormats.size(); i++) {
format.append(dateFormats.get(i));
if (i != dateFormats.size() - 1) {

21
jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticPropertyType.java → jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticPropertyType.java Normal file → Executable file
View File

@ -29,14 +29,17 @@ public enum ElasticPropertyType implements EnumDict<String> {
IP("ip", "ip", LongType::new),
ATTACHMENT("attachment", "attachment", FileType::new),
KEYWORD("string", "keyword", StringType::new),
GEO_POINT("geo_point", "geo_point", GeoType::new);
GEO_POINT("geo_point", "geo_point", GeoType::new),
GEO_SHAPE("geo_shape", "geo_shape", GeoShapeType::new)
;
@Getter
private String text;
private final String text;
@Getter
private String value;
private final String value;
private Supplier<DataType> typeBuilder;
private final Supplier<DataType> typeBuilder;
public DataType getType() {
return typeBuilder.get();
@ -53,14 +56,4 @@ public enum ElasticPropertyType implements EnumDict<String> {
return null;
}
public static ElasticPropertyType ofJava(Object value) {
if (!StringUtils.isEmpty(value)) {
for (ElasticPropertyType elasticPropertyType : ElasticPropertyType.values()) {
if (elasticPropertyType.getText().equals(value)) {
return elasticPropertyType;
}
}
}
throw new NotFoundException("未找到数据类型为：" + value + "的枚举");
}
}

26
jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticSearchTermType.java Normal file
View File

@ -0,0 +1,26 @@
package org.jetlinks.community.elastic.search.enums;
import co.elastic.clients.elasticsearch._types.query_dsl.Query;
import org.hswebframework.ezorm.core.param.Term;
import org.jetlinks.core.metadata.DataType;
import org.jetlinks.community.spi.Provider;
import org.jetlinks.community.things.utils.ThingsDatabaseUtils;
public interface ElasticSearchTermType {
Provider<ElasticSearchTermType> supports = Provider.create(ElasticSearchTermType.class);
String getId();
boolean isSupported(Term term);
Query.Builder process(Term term, Query.Builder builder);
default Query process(Term term) {
return process(term, new Query.Builder()).build();
}
default Object convertTermValue(DataType type, Object value) {
return ThingsDatabaseUtils.tryConvertTermValue(type, value);
}
}

41
jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticSearchTermTypes.java Normal file
View File

@ -0,0 +1,41 @@
package org.jetlinks.community.elastic.search.enums;
import org.hswebframework.ezorm.core.param.Term;
import java.util.Optional;
public class ElasticSearchTermTypes {
static {
}
public static void register(ElasticSearchTermType termType) {
ElasticSearchTermType.supports.register(termType.getId(), termType);
}
public static ElasticSearchTermType lookupNow(Term term) {
return lookup(term)
.orElseThrow(() -> new UnsupportedOperationException("不支持的查询条件:" + term.getType()));
}
public static Optional<ElasticSearchTermType> lookup(Term term) {
ElasticSearchTermType fast = ElasticSearchTermType
.supports
.get(term.getTermType())
.orElse(null);
if (fast != null) {
return Optional.of(fast);
}
for (ElasticSearchTermType termType : ElasticSearchTermType.supports.getAll()) {
if (termType.isSupported(term)) {
return Optional.of(termType);
}
}
return Optional.empty();
}
}

20
jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/DefaultElasticSearchIndexManager.java → jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/DefaultElasticSearchIndexManager.java
View File

@ -3,9 +3,12 @@ package org.jetlinks.community.elastic.search.index;
import lombok.Generated;
import lombok.Getter;
import lombok.Setter;
import org.apache.commons.collections4.map.CaseInsensitiveMap;
import org.jetlinks.core.cache.Caches;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import java.util.List;
@ -20,6 +23,12 @@ public class DefaultElasticSearchIndexManager implements ElasticSearchIndexManag
@Generated
private String defaultStrategy = "direct";
@Getter
@Setter
@Generated
//是否自动创建索引
private boolean autoCreate = true;
@Getter
@Setter
@Generated
@ -29,14 +38,23 @@ public class DefaultElasticSearchIndexManager implements ElasticSearchIndexManag
private final Map<String, ElasticSearchIndexMetadata> indexMetadataStore = new ConcurrentSkipListMap<>(String.CASE_INSENSITIVE_ORDER);
public DefaultElasticSearchIndexManager(@Autowired(required = false) List<ElasticSearchIndexStrategy> strategies) {
public DefaultElasticSearchIndexManager(@Autowired(required = false) List<ElasticSearchIndexStrategy> strategies,
@Autowired(required = false) List<ElasticSearchIndexCustomizer> customizers) {
if (strategies != null) {
strategies.forEach(this::registerStrategy);
}
if (customizers != null) {
customizers.forEach(customizer -> customizer.custom(this));
}
}
@Override
public Mono<Void> putIndex(ElasticSearchIndexMetadata index) {
//禁用索引创建
if (!autoCreate) {
indexMetadataStore.put(index.getIndex(), index);
return Mono.empty();
}
return this.getIndexStrategy(index.getIndex())
.flatMap(strategy -> strategy.putIndex(index))
.doOnNext(idx -> indexMetadataStore.put(idx.getIndex(), idx))

12
jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/DefaultElasticSearchIndexMetadata.java → jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/DefaultElasticSearchIndexMetadata.java Normal file → Executable file
View File

@ -1,5 +1,6 @@
package org.jetlinks.community.elastic.search.index;
import lombok.Generated;
import org.jetlinks.core.metadata.DataType;
import org.jetlinks.core.metadata.PropertyMetadata;
import org.jetlinks.core.metadata.SimplePropertyMetadata;
@ -10,9 +11,9 @@ import java.util.List;
import java.util.Map;
public class DefaultElasticSearchIndexMetadata implements ElasticSearchIndexMetadata {
private String index;
private final String index;
private Map<String, PropertyMetadata> properties = new HashMap<>();
private final Map<String, PropertyMetadata> properties = new HashMap<>();
public DefaultElasticSearchIndexMetadata(String index) {
this.index = index.toLowerCase().trim();
@ -24,16 +25,19 @@ public class DefaultElasticSearchIndexMetadata implements ElasticSearchIndexMeta
}
@Override
@Generated
public PropertyMetadata getProperty(String property) {
return properties.get(property);
}
@Override
@Generated
public String getIndex() {
return index;
}
@Override
@Generated
public List<PropertyMetadata> getProperties() {
return new ArrayList<>(properties.values());
}
@ -44,10 +48,10 @@ public class DefaultElasticSearchIndexMetadata implements ElasticSearchIndexMeta
}
public DefaultElasticSearchIndexMetadata addProperty(String property, DataType type) {
SimplePropertyMetadata metadata=new SimplePropertyMetadata();
SimplePropertyMetadata metadata = new SimplePropertyMetadata();
metadata.setValueType(type);
metadata.setId(property);
properties.put(property, metadata);
addProperty(metadata);
return this;
}
}

0
jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/ElasticIndex.java → jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/ElasticIndex.java Normal file → Executable file
View File

7
jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexCustomizer.java Executable file
View File

@ -0,0 +1,7 @@
package org.jetlinks.community.elastic.search.index;
public interface ElasticSearchIndexCustomizer {
void custom(ElasticSearchIndexManager manager);
}

15
jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexManager.java → jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexManager.java Normal file → Executable file
View File

@ -3,6 +3,12 @@ package org.jetlinks.community.elastic.search.index;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
/**
* ElasticSearch 索引管理器,用于统一管理,维护索引信息.
*
* @author zhouhao
* @since 1.0
*/
public interface ElasticSearchIndexManager {
/**
@ -23,6 +29,7 @@ public interface ElasticSearchIndexManager {
/**
* 获取多个所有元数据
*
* @param index 索引名称
* @return 索引元数据
*/
@ -41,7 +48,13 @@ public interface ElasticSearchIndexManager {
*/
Mono<ElasticSearchIndexStrategy> getIndexStrategy(String index);
default Flux<ElasticSearchIndexStrategy> getIndexesStrategy(String... index){
/**
* 获取多个索引的策略
*
* @param index 索引列表
* @return 索引策略
*/
default Flux<ElasticSearchIndexStrategy> getIndexesStrategy(String... index) {
return Flux
.fromArray(index)
.flatMap(this::getIndexStrategy);

2
jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexMetadata.java → jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexMetadata.java Normal file → Executable file
View File

@ -1,7 +1,7 @@
package org.jetlinks.community.elastic.search.index;
import org.jetlinks.community.elastic.search.utils.ElasticSearchConverter;
import org.jetlinks.core.metadata.PropertyMetadata;
import org.jetlinks.community.elastic.search.utils.ElasticSearchConverter;
import java.util.List;
import java.util.Map;

58
jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexProperties.java Executable file
View File

@ -0,0 +1,58 @@
package org.jetlinks.community.elastic.search.index;
import co.elastic.clients.elasticsearch.indices.IndexSettings;
import co.elastic.clients.json.JsonData;
import com.google.common.collect.Maps;
import lombok.*;
import org.apache.commons.collections4.MapUtils;
import org.jetlinks.community.elastic.search.ElasticSearchSupport;
import org.springframework.boot.context.properties.ConfigurationProperties;
import java.util.HashMap;
import java.util.Map;
@Getter
@Setter
@Builder
@NoArgsConstructor
@AllArgsConstructor
@ConfigurationProperties(prefix = "elasticsearch.index.settings")
@Generated
public class ElasticSearchIndexProperties {
//索引分片数据,通常为es的集群节点数量
private int numberOfShards = 1;
//副本数量
private int numberOfReplicas = 0;
//字段数量限制
private long totalFieldsLimit = 2000;
//默认字符串超过512将不会被索引,无法进行搜索
private int keywordIgnoreAbove = 512;
//其他的配置信息,在创建索引时将会设置到settings中
private Map<String, String> options;
//是否使用别名进行搜索
//设置为true将使用别名进行搜索,可通过手动绑定和接触别名来灵活配置搜索到的数据范围.
//设置为false时,将使用*进行搜索.在一些特殊请求,如索引名前缀类似时可能搜索到错误的数据.
private boolean useAliasSearch = true;
public IndexSettings.Builder toSettings(IndexSettings.Builder builder) {
return ElasticSearchSupport
.current()
.applyIndexSettings(this, builder);
}
public void addSetting(String key, String value) {
if (null == options) {
options = new HashMap<>();
}
options.put(key, value);
}
}

5
jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexStrategy.java → jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexStrategy.java Normal file → Executable file
View File

@ -41,5 +41,10 @@ public interface ElasticSearchIndexStrategy {
*/
Mono<ElasticSearchIndexMetadata> putIndex(ElasticSearchIndexMetadata metadata);
/**
* 加载索引元数据
* @param index 索引
* @return 索引元数据
*/
Mono<ElasticSearchIndexMetadata> loadIndexMetadata(String index);
}

277
jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/strategies/AbstractElasticSearchIndexStrategy.java Executable file
View File

@ -0,0 +1,277 @@
package org.jetlinks.community.elastic.search.index.strategies;
import co.elastic.clients.elasticsearch._types.mapping.DynamicTemplate;
import co.elastic.clients.elasticsearch._types.mapping.Property;
import co.elastic.clients.elasticsearch._types.mapping.TypeMapping;
import co.elastic.clients.elasticsearch.indices.PutMappingRequest;
import co.elastic.clients.elasticsearch.indices.get_mapping.IndexMappingRecord;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.jetlinks.core.metadata.DataType;
import org.jetlinks.core.metadata.PropertyMetadata;
import org.jetlinks.core.metadata.SimplePropertyMetadata;
import org.jetlinks.core.metadata.types.*;
import org.jetlinks.community.ConfigMetadataConstants;
import org.jetlinks.community.elastic.search.ElasticSearchSupport;
import org.jetlinks.community.elastic.search.enums.ElasticDateFormat;
import org.jetlinks.community.elastic.search.enums.ElasticPropertyType;
import org.jetlinks.community.elastic.search.index.DefaultElasticSearchIndexMetadata;
import org.jetlinks.community.elastic.search.index.ElasticSearchIndexMetadata;
import org.jetlinks.community.elastic.search.index.ElasticSearchIndexProperties;
import org.jetlinks.community.elastic.search.index.ElasticSearchIndexStrategy;
import org.jetlinks.community.elastic.search.service.reactive.ReactiveElasticsearchClient;
import org.jetlinks.reactor.ql.utils.CastUtils;
import org.springframework.util.CollectionUtils;
import reactor.core.publisher.Mono;
import java.util.*;
import java.util.stream.Collectors;
@AllArgsConstructor
@Slf4j
public abstract class AbstractElasticSearchIndexStrategy implements ElasticSearchIndexStrategy {
@Getter
private final String id;
protected ReactiveElasticsearchClient client;
protected ElasticSearchIndexProperties properties;
protected String wrapIndex(String index) {
return index.toLowerCase();
}
protected Mono<Boolean> indexExists(String index) {
return client.execute(
c -> c
.indices()
.exists(b -> b.index(index)).value());
}
protected Mono<Void> doCreateIndex(ElasticSearchIndexMetadata metadata) {
return client
.execute(c -> {
c.indices()
.create(builder -> createIndexRequest(builder, metadata));
return null;
});
}
protected Mono<Void> doPutIndex(ElasticSearchIndexMetadata metadata,
boolean justUpdateMapping) {
String index = wrapIndex(metadata.getIndex());
return this.indexExists(index)
.flatMap(exists -> {
if (exists) {
return doLoadIndexMetadata(index)
.flatMap(oldMapping -> {
PutMappingRequest.Builder builder =
createPutMappingRequest(metadata, oldMapping, new PutMappingRequest.Builder());
//无需更新
if (builder == null) {
return Mono.empty();
}
return client
.execute(c -> c.indices().putMapping(builder.build()));
})
.then();
}
if (justUpdateMapping) {
return Mono.empty();
}
return doCreateIndex(metadata);
});
}
protected Mono<ElasticSearchIndexMetadata> doLoadIndexMetadata(String _index) {
String index = wrapIndex(_index);
return client
.execute(c -> {
IndexMappingRecord record =
ElasticSearchSupport
.current()
.getIndexMapping(
c.indices()
.getMapping(b -> b
.ignoreUnavailable(true)
.allowNoIndices(true)
.index(index)),
index
);
if (record == null || record.mappings() == null) {
return null;
}
return convertMetadata(index, record.mappings());
});
}
protected co.elastic.clients.elasticsearch.indices.CreateIndexRequest.Builder
createIndexRequest(co.elastic.clients.elasticsearch.indices.CreateIndexRequest.Builder builder,
ElasticSearchIndexMetadata metadata) {
builder.index(wrapIndex(metadata.getIndex()));
builder.settings(properties::toSettings);
builder.mappings(b -> {
b.properties(createElasticProperties(metadata.getProperties()));
b.dynamicTemplates(createDynamicTemplates());
return b;
});
return builder;
}
private co.elastic.clients.elasticsearch.indices.PutMappingRequest.Builder
createPutMappingRequest(ElasticSearchIndexMetadata metadata,
ElasticSearchIndexMetadata ignore,
co.elastic.clients.elasticsearch.indices.PutMappingRequest.Builder builder) {
Map<String, Property> properties = createElasticProperties(metadata.getProperties());
Map<String, Property> ignoreProperties = createElasticProperties(ignore.getProperties());
for (Map.Entry<String, Property> en : ignoreProperties.entrySet()) {
log.trace("ignore update index [{}] mapping property:{},{}", wrapIndex(metadata.getIndex()), en.getKey(), en
.getValue());
properties.remove(en.getKey());
}
if (properties.isEmpty()) {
log.debug("ignore update index [{}] mapping", wrapIndex(metadata.getIndex()));
return null;
}
List<PropertyMetadata> allProperties = new ArrayList<>();
allProperties.addAll(metadata.getProperties());
allProperties.addAll(ignore.getProperties());
builder.index(wrapIndex(metadata.getIndex()));
builder.properties(createElasticProperties(allProperties));
return builder;
}
protected Map<String, Property> createElasticProperties(List<PropertyMetadata> metadata) {
if (metadata == null) {
return new HashMap<>();
}
return metadata
.stream()
.collect(Collectors.toMap(PropertyMetadata::getId,
prop -> this.createElasticProperty(prop.getValueType()), (a, v) -> a));
}
protected Property createElasticProperty(DataType type) {
if (type instanceof DateTimeType) {
return Property.of(b -> b
.date(b2 -> b2
.format(
ElasticDateFormat.getFormat(
ElasticDateFormat.epoch_millis,
ElasticDateFormat.strict_date_hour_minute_second,
ElasticDateFormat.strict_date_time,
ElasticDateFormat.strict_date)
)));
} else if (type instanceof DoubleType) {
return Property.of(b -> b.double_(b2 -> b2.nullValue(null)));
} else if (type instanceof LongType) {
return Property.of(b -> b.long_(b2 -> b2.nullValue(null)));
} else if (type instanceof IntType) {
return Property.of(b -> b.integer(b2 -> b2.nullValue(null)));
} else if (type instanceof FloatType) {
return Property.of(b -> b.float_(b2 -> b2.nullValue(null)));
} else if (type instanceof BooleanType) {
return Property.of(b -> b.boolean_(b2 -> b2.nullValue(null)));
} else if (type instanceof GeoType) {
return Property.of(b -> b.geoPoint(b2 -> b2));
} else if (type instanceof GeoShapeType) {
return Property.of(b -> b.geoShape(b2 -> b2));
} else if (type instanceof ArrayType) {
return createElasticProperty(((ArrayType) type).getElementType());
} else if (type instanceof ObjectType objectType) {
if (!CollectionUtils.isEmpty(objectType.getProperties())) {
return Property.of(b -> b
.nested(b2 -> b2.properties(createElasticProperties(objectType.getProperties()))));
}
return Property.of(b -> b.nested(b2 -> b2));
} else {
int above = Optional
.ofNullable(type)
.flatMap(_type -> _type.getExpand(ConfigMetadataConstants.maxLength.getKey()))
.filter(val -> val instanceof Number || StringUtils.isNumeric(String.valueOf(val)))
.map(CastUtils::castNumber)
.map(Number::intValue)
.orElse(properties.getKeywordIgnoreAbove());
return Property.of(b -> b.keyword(b2 -> b2.ignoreAbove(above)));
}
}
protected ElasticSearchIndexMetadata convertMetadata(String index, TypeMapping mapping) {
Map<String, Property> properties = mapping.properties();
return new DefaultElasticSearchIndexMetadata(index, convertProperties(properties));
}
@SuppressWarnings("all")
protected List<PropertyMetadata> convertProperties(Map<String, Property> properties) {
return properties
.entrySet()
.stream()
.map(entry -> convertProperty(entry.getKey(), entry.getValue()))
.collect(Collectors.toList());
}
private PropertyMetadata convertProperty(String property, Property prop) {
SimplePropertyMetadata metadata = new SimplePropertyMetadata();
metadata.setId(property);
metadata.setName(property);
ElasticPropertyType elasticPropertyType = ElasticPropertyType.of(prop._kind().jsonValue());
if (null != elasticPropertyType) {
DataType dataType = elasticPropertyType.getType();
if ((elasticPropertyType == ElasticPropertyType.OBJECT
|| elasticPropertyType == ElasticPropertyType.NESTED)
&& dataType instanceof ObjectType) {
ObjectType objectType = ((ObjectType) dataType);
objectType.setProperties(convertProperties(prop.nested().properties()));
}
metadata.setValueType(dataType);
} else {
metadata.setValueType(StringType.GLOBAL);
}
return metadata;
}
protected List<Map<String, DynamicTemplate>> createDynamicTemplates() {
List<Map<String, DynamicTemplate>> list = new ArrayList<>();
{
list.add(Collections.singletonMap(
"string_fields", ElasticSearchSupport
.current()
.createDynamicTemplate(
"string",
createElasticProperty(StringType.GLOBAL))));
}
{
list.add(Collections.singletonMap(
"date_fields", ElasticSearchSupport
.current()
.createDynamicTemplate(
"string",
createElasticProperty(StringType.GLOBAL))));
}
return list;
}
}

86
jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/strategies/AffixesElasticSearchIndexStrategy.java Normal file
View File

@ -0,0 +1,86 @@
package org.jetlinks.community.elastic.search.index.strategies;
import lombok.Getter;
import lombok.Setter;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.jetlinks.community.elastic.search.index.ElasticSearchIndexMetadata;
import org.jetlinks.community.elastic.search.index.ElasticSearchIndexProperties;
import org.jetlinks.community.elastic.search.service.reactive.ReactiveElasticsearchClient;
import org.springframework.boot.context.properties.ConfigurationProperties;
import reactor.core.publisher.Mono;
/**
* 前后缀索引策略支持.
* <pre>{@code
*
* elasticsearch:
* index:
* default-strategy: affixes
* affixes:
* prefix: "" #前缀
* suffix: "_test" # 后缀
* auto-create: false # 是否创建索引
*
* }</pre>
*
* @author zhouhao
* @since 2.2
*/
@Getter
@Setter
@Slf4j
@ConfigurationProperties(prefix = "elasticsearch.index.affixes")
public class AffixesElasticSearchIndexStrategy extends AbstractElasticSearchIndexStrategy {
//前缀
private String prefix = "";
//后缀
private String suffix = "";
//是否自动创建索引
private boolean autoCreate = true;
public AffixesElasticSearchIndexStrategy(ReactiveElasticsearchClient client,
ElasticSearchIndexProperties properties) {
super("affixes", client, properties);
}
@Override
public String getIndexForSave(String index) {
return prefix + index + suffix;
}
@Override
public String getIndexForSearch(String index) {
return prefix + index + suffix;
}
@Override
@SneakyThrows
public Mono<ElasticSearchIndexMetadata> putIndex(ElasticSearchIndexMetadata metadata) {
if (log.isInfoEnabled() && !autoCreate) {
// CreateIndexRequest request = createIndexRequest(metadata);
// Object data = ObjectMappers
// .parseJson(
// new RequestCreator() {
// }
// .createIndexRequest().apply(request).getEntity()
// .getContent(), Object.class);
// log.info("ignore put elasticsearch index [{}] :\n{}", metadata.getIndex(), JSON.toJSONString(data, SerializerFeature.PrettyFormat));
}
if (autoCreate) {
return this
.doPutIndex(metadata.newIndexName(getIndexForSave(metadata.getIndex())), false)
.thenReturn(metadata);
}
return Mono.just(metadata);
}
@Override
public Mono<ElasticSearchIndexMetadata> loadIndexMetadata(String index) {
return doLoadIndexMetadata(getIndexForSearch(index));
}
}

0
jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/strategies/DirectElasticSearchIndexStrategy.java → jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/strategies/DirectElasticSearchIndexStrategy.java
View File

74
jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/strategies/TemplateElasticSearchIndexStrategy.java → jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/strategies/TemplateElasticSearchIndexStrategy.java
View File

@ -1,20 +1,16 @@
package org.jetlinks.community.elastic.search.index.strategies;
import co.elastic.clients.elasticsearch.indices.PutIndexTemplateRequest;
import co.elastic.clients.elasticsearch.indices.TemplateMapping;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.elasticsearch.Version;
import org.elasticsearch.action.admin.indices.alias.Alias;
import org.elasticsearch.client.indices.GetIndexTemplatesRequest;
import org.elasticsearch.client.indices.PutIndexTemplateRequest;
import org.jetlinks.community.elastic.search.ElasticSearchSupport;
import org.jetlinks.community.elastic.search.index.ElasticSearchIndexMetadata;
import org.jetlinks.community.elastic.search.index.ElasticSearchIndexProperties;
import org.jetlinks.community.elastic.search.service.reactive.ReactiveElasticsearchClient;
import reactor.core.publisher.Mono;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Slf4j
public abstract class TemplateElasticSearchIndexStrategy extends AbstractElasticSearchIndexStrategy {
@ -40,15 +36,21 @@ public abstract class TemplateElasticSearchIndexStrategy extends AbstractElastic
@Override
public String getIndexForSearch(String index) {
return getAlias(index);
if (properties.isUseAliasSearch()) {
return getAlias(index);
} else {
return wrapIndex(index).concat("*");
}
}
@Override
public Mono<ElasticSearchIndexMetadata> putIndex(ElasticSearchIndexMetadata metadata) {
String saveIndex = getIndexForSave(metadata.getIndex());
return client
.putTemplate(createIndexTemplateRequest(metadata))
//修改当前索引
.execute(c -> c
.indices()
.putIndexTemplate(request -> createIndexTemplateRequest(request, metadata)))
.then(doPutIndex(metadata.newIndexName(saveIndex), true)
//忽略修改索引错误
.onErrorResume(err -> {
@ -58,32 +60,44 @@ public abstract class TemplateElasticSearchIndexStrategy extends AbstractElastic
.thenReturn(metadata.newIndexName(wrapIndex(metadata.getIndex())));
}
protected PutIndexTemplateRequest createIndexTemplateRequest(ElasticSearchIndexMetadata metadata) {
protected PutIndexTemplateRequest.Builder createIndexTemplateRequest(PutIndexTemplateRequest.Builder builder,
ElasticSearchIndexMetadata metadata) {
String index = wrapIndex(metadata.getIndex());
PutIndexTemplateRequest request = new PutIndexTemplateRequest(getTemplate(index));
request.alias(new Alias(getAlias(index)));
request.settings(properties.toSettings());
Map<String, Object> mappingConfig = new HashMap<>();
mappingConfig.put("properties", createElasticProperties(metadata.getProperties()));
mappingConfig.put("dynamic_templates", createDynamicTemplates());
mappingConfig.put("_source", Collections.singletonMap("enabled", true));
if (client.serverVersion().after(Version.V_7_0_0)) {
request.mapping(mappingConfig);
} else {
request.mapping(Collections.singletonMap("_doc", mappingConfig));
builder.name(getTemplate(index));
builder.indexPatterns(getIndexPatterns(index));
// 7.x不支持此设置
if (ElasticSearchSupport.current().is8x()) {
builder.allowAutoCreate(true);
}
request.patterns(getIndexPatterns(index));
return request;
builder.template(template -> {
template.aliases(getAlias(index), a -> a);
template.mappings(mapping -> {
mapping.dynamicTemplates(createDynamicTemplates());
mapping.properties(createElasticProperties(metadata.getProperties()));
mapping.source(s -> s.enabled(true));
return mapping;
});
return template;
});
return builder;
}
@Override
public Mono<ElasticSearchIndexMetadata> loadIndexMetadata(String index) {
return client.getTemplate(new GetIndexTemplatesRequest(getTemplate(index)))
.filter(resp -> CollectionUtils.isNotEmpty(resp.getIndexTemplates()))
.flatMap(resp -> Mono.justOrEmpty(convertMetadata(index, resp
.getIndexTemplates()
.get(0)
.mappings())));
String name = getTemplate(index);
return client.execute(t -> {
TemplateMapping mapping = ElasticSearchSupport
.current()
.getTemplateMapping(
t.indices()
.getTemplate(request -> request.name(getTemplate(index))),
name
);
return mapping == null ? null : convertMetadata(index, mapping.mappings());
});
}
}

19
jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/strategies/TimeByDayElasticSearchIndexStrategy.java → jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/strategies/TimeByDayElasticSearchIndexStrategy.java Normal file → Executable file
View File

@ -1,32 +1,29 @@
package org.jetlinks.community.elastic.search.index.strategies;
import org.hswebframework.utils.time.DateFormatter;
import org.jetlinks.community.elastic.search.index.ElasticSearchIndexProperties;
import org.jetlinks.community.elastic.search.service.reactive.ReactiveElasticsearchClient;
import org.springframework.stereotype.Component;
import java.time.Clock;
import java.time.LocalDate;
import java.util.Date;
/**
* 按日期来划分索引策略
* 按天来划分索引策略
*
* @author caizz
* @since 1.0
* @author zhouhao
* @since 2.2
*/
@Component
public class TimeByDayElasticSearchIndexStrategy extends TemplateElasticSearchIndexStrategy {
private static final Clock CLOCK = Clock.systemDefaultZone();
public TimeByDayElasticSearchIndexStrategy(ReactiveElasticsearchClient client, ElasticSearchIndexProperties properties) {
super("time-by-day", client, properties);
}
@Override
public String getIndexForSave(String index) {
LocalDate now = LocalDate.now();
LocalDate now = LocalDate.now(CLOCK);
String idx = wrapIndex(index);
return idx + "_" + now.getYear()
+ "-" + (now.getMonthValue() < 10 ? "0" : "") + now.getMonthValue()
+ "-" + (now.getDayOfMonth() < 10 ? "0" : "") + now.getDayOfMonth();
return idx + "_" + now.getYear() + "-" + now.getMonthValue() + "-" + now.getDayOfMonth();
}
}

5
jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/strategies/TimeByMonthElasticSearchIndexStrategy.java → jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/strategies/TimeByMonthElasticSearchIndexStrategy.java
View File

@ -3,6 +3,7 @@ package org.jetlinks.community.elastic.search.index.strategies;
import org.jetlinks.community.elastic.search.index.ElasticSearchIndexProperties;
import org.jetlinks.community.elastic.search.service.reactive.ReactiveElasticsearchClient;
import java.time.Clock;
import java.time.LocalDate;
/**
@ -13,13 +14,15 @@ import java.time.LocalDate;
*/
public class TimeByMonthElasticSearchIndexStrategy extends TemplateElasticSearchIndexStrategy {
private static final Clock CLOCK = Clock.systemDefaultZone();
public TimeByMonthElasticSearchIndexStrategy(ReactiveElasticsearchClient client, ElasticSearchIndexProperties properties) {
super("time-by-month", client, properties);
}
@Override
public String getIndexForSave(String index) {
LocalDate now = LocalDate.now();
LocalDate now = LocalDate.now(CLOCK);
String idx = wrapIndex(index);
return idx + "_" + now.getYear() + "-" + now.getMonthValue();
}

Some files were not shown because too many files have changed in this diff Show More