suguo
/
jetlinks-community
mirror of https://gitee.com/jetlinks/jetlinks-community.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

升级elasticsearch 7.9.0

This commit is contained in:
zhou-hao 2020-09-01 18:20:26 +08:00
parent d793bb5391
commit 75c7022ebf
21 changed files with 2183 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docker/run-all/docker-compose.yml
View File

@ -59,7 +59,7 @@ services:
links:
- jetlinks:jetlinks
jetlinks:
image: registry.cn-shenzhen.aliyuncs.com/jetlinks/jetlinks-standalone:1.4.0
image: registry.cn-shenzhen.aliyuncs.com/jetlinks/jetlinks-standalone:1.5.0-SNAPSHOT
container_name: jetlinks-ce
ports:
- 8848:8848 # API端口

2
jetlinks-components/common-component/src/main/java/org/jetlinks/community/Version.java
View File

@ -8,6 +8,6 @@ public class Version {
private final String edition = "community";
private final String version = "1.4.0";
private final String version = "1.5.0-SNAPSHOT";
}

13
jetlinks-components/elasticsearch-component/pom.xml
View File

@ -43,6 +43,12 @@
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.data</groupId>
<artifactId>spring-data-elasticsearch</artifactId>
</dependency>
<dependency>
<groupId>org.hswebframework.web</groupId>
<artifactId>hsweb-commons-crud</artifactId>
@ -65,7 +71,12 @@
<artifactId>timeseries-component</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>io.projectreactor.netty</groupId>
<artifactId>reactor-netty</artifactId>
</dependency>
</dependencies>
</project>

2
jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/DefaultAggregationService.java
View File

@ -39,7 +39,7 @@ import java.util.stream.Collectors;
* @author bsetfeng
* @since 1.0
**/
@Service
//@Service
@Slf4j
public class DefaultAggregationService implements AggregationService {

7
jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/bucket/AggregationResponseHandle.java
View File

@ -4,12 +4,7 @@ import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import org.elasticsearch.search.aggregations.bucket.range.Range;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.metrics.avg.Avg;
import org.elasticsearch.search.aggregations.metrics.max.Max;
import org.elasticsearch.search.aggregations.metrics.min.Min;
import org.elasticsearch.search.aggregations.metrics.stats.Stats;
import org.elasticsearch.search.aggregations.metrics.sum.Sum;
import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCount;
import org.elasticsearch.search.aggregations.metrics.*;
import org.jetlinks.community.elastic.search.aggreation.metrics.MetricsResponseSingleValue;
import java.util.List;

8
jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/enums/BucketType.java
View File

@ -17,9 +17,9 @@ import org.jetlinks.community.elastic.search.aggreation.bucket.Bucket;
import org.jetlinks.community.elastic.search.aggreation.bucket.BucketAggregationsStructure;
import org.jetlinks.community.elastic.search.aggreation.bucket.Sort;
import org.jetlinks.community.elastic.search.aggreation.metrics.MetricsAggregationStructure;
import org.joda.time.DateTimeZone;
import org.springframework.util.StringUtils;
import java.time.ZoneId;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -104,7 +104,7 @@ public enum BucketType {
if (structure.getMissingValue() != null) {
builder.missing(structure.getMissingValue());
}
builder.timeZone(DateTimeZone.getDefault());
builder.timeZone(ZoneId.systemDefault());
commonAggregationSetting(builder, structure);
return builder;
}
@ -136,7 +136,7 @@ public enum BucketType {
if (sort != null) {
builder.order(mapping.get(OrderBuilder.of(sort.getOrder(), sort.getType())));
}
builder.timeZone(DateTimeZone.getDefault());
builder.timeZone(ZoneId.systemDefault());
commonAggregationSetting(builder, structure);
return builder;
}
@ -148,7 +148,7 @@ public enum BucketType {
}
};
private String text;
private final String text;
public abstract AggregationBuilder aggregationBuilder(BucketAggregationsStructure structure);

7
jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/enums/MetricsType.java
View File

@ -5,12 +5,7 @@ import lombok.Getter;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.search.aggregations.AggregationBuilder;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.metrics.avg.Avg;
import org.elasticsearch.search.aggregations.metrics.max.Max;
import org.elasticsearch.search.aggregations.metrics.min.Min;
import org.elasticsearch.search.aggregations.metrics.stats.Stats;
import org.elasticsearch.search.aggregations.metrics.sum.Sum;
import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCount;
import org.elasticsearch.search.aggregations.metrics.*;
import org.jetlinks.community.elastic.search.aggreation.metrics.MetricsResponse;
import org.jetlinks.community.elastic.search.aggreation.metrics.MetricsResponseSingleValue;

106
jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/configuration/ElasticSearchConfiguration.java
View File

@ -1,5 +1,12 @@
package org.jetlinks.community.elastic.search.configuration;
import io.netty.channel.ChannelOption;
import io.netty.handler.ssl.ApplicationProtocolConfig;
import io.netty.handler.ssl.ClientAuth;
import io.netty.handler.ssl.IdentityCipherSuiteFilter;
import io.netty.handler.ssl.JdkSslContext;
import io.netty.handler.timeout.ReadTimeoutHandler;
import io.netty.handler.timeout.WriteTimeoutHandler;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.client.RestClient;
@ -8,10 +15,25 @@ import org.jetlinks.community.elastic.search.ElasticRestClient;
import org.jetlinks.community.elastic.search.embedded.EmbeddedElasticSearch;
import org.jetlinks.community.elastic.search.embedded.EmbeddedElasticSearchProperties;
import org.jetlinks.community.elastic.search.index.ElasticSearchIndexProperties;
import org.jetlinks.community.elastic.search.service.reactive.DefaultReactiveElasticsearchClient;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.elasticsearch.client.ClientConfiguration;
import org.springframework.data.elasticsearch.client.reactive.HostProvider;
import org.springframework.data.elasticsearch.client.reactive.RequestCreator;
import org.springframework.data.elasticsearch.client.reactive.WebClientProvider;
import org.springframework.http.client.reactive.ReactorClientHttpConnector;
import reactor.netty.http.client.HttpClient;
import reactor.netty.tcp.ProxyProvider;
import reactor.netty.tcp.TcpClient;
import javax.net.ssl.SSLContext;
import java.net.InetSocketAddress;
import java.time.Duration;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
/**
* @author bsetfeng
@ -34,18 +56,94 @@ public class ElasticSearchConfiguration {
this.properties = properties;
this.embeddedProperties = embeddedProperties;
}
@Bean
@SneakyThrows
public ElasticRestClient elasticRestClient() {
public DefaultReactiveElasticsearchClient reactiveElasticsearchClient(ClientConfiguration clientConfiguration) {
if (embeddedProperties.isEnabled()) {
log.debug("starting embedded elasticsearch on {}:{}",
embeddedProperties.getHost(),
embeddedProperties.getPort());
new EmbeddedElasticSearch(embeddedProperties)
.start();
new EmbeddedElasticSearch(embeddedProperties).start();
}
WebClientProvider provider = getWebClientProvider(clientConfiguration);
HostProvider hostProvider = HostProvider.provider(provider, clientConfiguration.getHeadersSupplier(),
clientConfiguration.getEndpoints().toArray(new InetSocketAddress[0]));
DefaultReactiveElasticsearchClient client =
new DefaultReactiveElasticsearchClient(hostProvider, new RequestCreator() {
});
client.setHeadersSupplier(clientConfiguration.getHeadersSupplier());
return client;
}
private static WebClientProvider getWebClientProvider(ClientConfiguration clientConfiguration) {
Duration connectTimeout = clientConfiguration.getConnectTimeout();
Duration soTimeout = clientConfiguration.getSocketTimeout();
TcpClient tcpClient = TcpClient.create();
if (!connectTimeout.isNegative()) {
tcpClient = tcpClient.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, Math.toIntExact(connectTimeout.toMillis()));
}
if (!soTimeout.isNegative()) {
tcpClient = tcpClient.doOnConnected(connection -> connection //
.addHandlerLast(new ReadTimeoutHandler(soTimeout.toMillis(), TimeUnit.MILLISECONDS))
.addHandlerLast(new WriteTimeoutHandler(soTimeout.toMillis(), TimeUnit.MILLISECONDS)));
}
if (clientConfiguration.getProxy().isPresent()) {
String proxy = clientConfiguration.getProxy().get();
String[] hostPort = proxy.split(":");
if (hostPort.length != 2) {
throw new IllegalArgumentException("invalid proxy configuration " + proxy + ", should be \"host:port\"");
}
tcpClient = tcpClient.proxy(proxyOptions -> proxyOptions.type(ProxyProvider.Proxy.HTTP).host(hostPort[0])
.port(Integer.parseInt(hostPort[1])));
}
String scheme = "http";
HttpClient httpClient = HttpClient.from(tcpClient);
if (clientConfiguration.useSsl()) {
Optional<SSLContext> sslContext = clientConfiguration.getSslContext();
if (sslContext.isPresent()) {
httpClient = httpClient.secure(sslContextSpec -> {
sslContextSpec.sslContext(new JdkSslContext(sslContext.get(), true, null, IdentityCipherSuiteFilter.INSTANCE,
ApplicationProtocolConfig.DISABLED, ClientAuth.NONE, null, false));
});
} else {
httpClient = httpClient.secure();
}
scheme = "https";
}
ReactorClientHttpConnector connector = new ReactorClientHttpConnector(httpClient);
WebClientProvider provider = WebClientProvider.create(scheme, connector);
if (clientConfiguration.getPathPrefix() != null) {
provider = provider.withPathPrefix(clientConfiguration.getPathPrefix());
}
provider = provider.withDefaultHeaders(clientConfiguration.getDefaultHeaders()) //
.withWebClientConfigurer(clientConfiguration.getWebClientConfigurer());
return provider;
}
@Bean
@SneakyThrows
public ElasticRestClient elasticRestClient() {
RestHighLevelClient client = new RestHighLevelClient(RestClient.builder(properties.createHosts())
.setRequestConfigCallback(properties::applyRequestConfigBuilder)
.setHttpClientConfigCallback(properties::applyHttpAsyncClientBuilder));

11
jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/embedded/EmbeddedElasticSearch.java
View File

@ -22,17 +22,12 @@ public class EmbeddedElasticSearch extends Node {
Settings.builder()
.put("node.name", "test")
.put("discovery.type", "single-node")
.put("transport.type", "netty4")
.put("http.type", "netty4")
.put("transport.type", Netty4Plugin.NETTY_TRANSPORT_NAME)
.put("http.type", Netty4Plugin.NETTY_HTTP_TRANSPORT_NAME)
.put("network.host", "0.0.0.0")
.put("http.port", 9200)
).build(), null),
).build(), Collections.emptyMap(), null, () -> "default"),
Collections.singleton(Netty4Plugin.class), false);
}
@Override
protected void registerDerivedNodeNameWithLogger(String nodeName) {
}
}

71
jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/strategies/AbstractElasticSearchIndexStrategy.java
View File

@ -3,22 +3,23 @@ package org.jetlinks.community.elastic.search.index.strategies;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.indices.*;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.jetlinks.community.elastic.search.index.ElasticSearchIndexProperties;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
import org.elasticsearch.cluster.metadata.MappingMetadata;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.compress.CompressedXContent;
import org.jetlinks.core.metadata.DataType;
import org.jetlinks.core.metadata.PropertyMetadata;
import org.jetlinks.core.metadata.SimplePropertyMetadata;
import org.jetlinks.core.metadata.types.*;
import org.jetlinks.community.elastic.search.ElasticRestClient;
import org.jetlinks.community.elastic.search.enums.ElasticDateFormat;
import org.jetlinks.community.elastic.search.enums.ElasticPropertyType;
import org.jetlinks.community.elastic.search.index.DefaultElasticSearchIndexMetadata;
import org.jetlinks.community.elastic.search.index.ElasticSearchIndexMetadata;
import org.jetlinks.community.elastic.search.index.ElasticSearchIndexProperties;
import org.jetlinks.community.elastic.search.index.ElasticSearchIndexStrategy;
import org.jetlinks.community.elastic.search.utils.ReactorActionListener;
import org.jetlinks.community.elastic.search.service.reactive.ReactiveElasticsearchClient;
import reactor.core.publisher.Mono;
import java.util.*;
@ -30,7 +31,7 @@ public abstract class AbstractElasticSearchIndexStrategy implements ElasticSearc
@Getter
private final String id;
protected ElasticRestClient client;
protected ReactiveElasticsearchClient client;
protected ElasticSearchIndexProperties properties;
@ -39,19 +40,11 @@ public abstract class AbstractElasticSearchIndexStrategy implements ElasticSearc
}
protected Mono<Boolean> indexExists(String index) {
return ReactorActionListener.mono(
actionListener ->
client.getQueryClient()
.indices()
.existsAsync(new GetIndexRequest(wrapIndex(index)), RequestOptions.DEFAULT, actionListener));
return client.existsIndex(req -> req.indices(wrapIndex(index)));
}
protected Mono<Void> doCreateIndex(ElasticSearchIndexMetadata metadata) {
return ReactorActionListener.<CreateIndexResponse>mono(
actionListener -> client.getQueryClient()
.indices()
.createAsync(createIndexRequest(metadata), RequestOptions.DEFAULT, actionListener))
.then();
return client.createIndex(createIndexRequest(metadata));
}
protected Mono<Void> doPutIndex(ElasticSearchIndexMetadata metadata,
@ -62,11 +55,7 @@ public abstract class AbstractElasticSearchIndexStrategy implements ElasticSearc
if (exists) {
return doLoadIndexMetadata(index)
.flatMap(oldMapping -> Mono.justOrEmpty(createPutMappingRequest(metadata, oldMapping)))
.flatMap(request -> ReactorActionListener.<AcknowledgedResponse>mono(
actionListener ->
client.getWriteClient()
.indices()
.putMappingAsync(request, RequestOptions.DEFAULT, actionListener)))
.flatMap(request -> client.updateMapping(request))
.then();
}
if (justUpdateMapping) {
@ -78,21 +67,18 @@ public abstract class AbstractElasticSearchIndexStrategy implements ElasticSearc
protected Mono<ElasticSearchIndexMetadata> doLoadIndexMetadata(String _index) {
String index = wrapIndex(_index);
return ReactorActionListener
.<GetMappingsResponse>mono(listener -> client.getQueryClient()
.indices()
.getMappingAsync(new GetMappingsRequest().indices(index), RequestOptions.DEFAULT, listener))
return client.getMapping(new GetMappingsRequest().indices(index))
.flatMap(resp -> Mono.justOrEmpty(convertMetadata(index, resp.mappings().get(index))));
}
public CreateIndexRequest createIndexRequest(ElasticSearchIndexMetadata metadata) {
protected CreateIndexRequest createIndexRequest(ElasticSearchIndexMetadata metadata) {
CreateIndexRequest request = new CreateIndexRequest(wrapIndex(metadata.getIndex()));
request.settings(properties.toSettings());
Map<String, Object> mappingConfig = new HashMap<>();
mappingConfig.put("properties", createElasticProperties(metadata.getProperties()));
mappingConfig.put("dynamic_templates", createDynamicTemplates());
request.mapping(mappingConfig);
mappingConfig.forEach(request::mapping);
return request;
}
@ -107,8 +93,10 @@ public abstract class AbstractElasticSearchIndexStrategy implements ElasticSearc
log.debug("ignore update index [{}] mapping", wrapIndex(metadata.getIndex()));
return null;
}
Map<String, Object> mappingConfig = new HashMap<>();
PutMappingRequest request = new PutMappingRequest(wrapIndex(metadata.getIndex()));
request.source(Collections.singletonMap("properties", properties));
mappingConfig.put("properties", createElasticProperties(metadata.getProperties()));
request.source(mappingConfig);
return request;
}
@ -142,6 +130,8 @@ public abstract class AbstractElasticSearchIndexStrategy implements ElasticSearc
property.put("type", "boolean");
} else if (type instanceof GeoType) {
property.put("type", "geo_point");
} else if (type instanceof GeoShapeType) {
property.put("type", "geo_shape");
} else if (type instanceof ArrayType) {
ArrayType arrayType = ((ArrayType) type);
return createElasticProperty(arrayType.getElementType());
@ -151,14 +141,26 @@ public abstract class AbstractElasticSearchIndexStrategy implements ElasticSearc
property.put("properties", createElasticProperties(objectType.getProperties()));
} else {
property.put("type", "keyword");
property.put("ignore_above",512);
property.put("ignore_above", 512);
}
return property;
}
protected ElasticSearchIndexMetadata convertMetadata(String index, MappingMetaData metaData) {
Map<String, Object> response = metaData.getSourceAsMap();
Object properties = response.get("properties");
protected ElasticSearchIndexMetadata convertMetadata(String index, ImmutableOpenMap<String, ?> metaData) {
MappingMetadata mappingMetadata = null;
if (metaData.size() == 1) {
Object res = metaData.values().iterator().next().value;
if (res instanceof MappingMetadata) {
mappingMetadata = ((MappingMetadata) res);
} else if (res instanceof CompressedXContent) {
mappingMetadata = new MappingMetadata(((CompressedXContent) res));
}
}
if (mappingMetadata == null) {
throw new UnsupportedOperationException("unsupported index metadata" + metaData);
}
Object properties = mappingMetadata.getSourceAsMap().get("properties");
return new DefaultElasticSearchIndexMetadata(index, convertProperties(properties));
}
@ -216,5 +218,4 @@ public abstract class AbstractElasticSearchIndexStrategy implements ElasticSearc
return maps;
}
}

6
jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/strategies/DirectElasticSearchIndexStrategy.java
View File

@ -1,16 +1,16 @@
package org.jetlinks.community.elastic.search.index.strategies;
import org.jetlinks.community.elastic.search.ElasticRestClient;
import org.jetlinks.community.elastic.search.index.ElasticSearchIndexMetadata;
import org.jetlinks.community.elastic.search.index.ElasticSearchIndexProperties;
import org.jetlinks.community.elastic.search.service.reactive.ReactiveElasticsearchClient;
import org.springframework.stereotype.Component;
import reactor.core.publisher.Mono;
@Component
public class DirectElasticSearchIndexStrategy extends AbstractElasticSearchIndexStrategy {
public DirectElasticSearchIndexStrategy(ElasticRestClient client, ElasticSearchIndexProperties properties) {
super("direct", client,properties);
public DirectElasticSearchIndexStrategy(ReactiveElasticsearchClient client, ElasticSearchIndexProperties properties) {
super("direct", client, properties);
}
@Override

29
jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/strategies/TemplateElasticSearchIndexStrategy.java
View File

@ -1,15 +1,11 @@
package org.jetlinks.community.elastic.search.index.strategies;
import org.elasticsearch.action.admin.indices.alias.Alias;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.indices.GetIndexTemplatesRequest;
import org.elasticsearch.client.indices.GetIndexTemplatesResponse;
import org.elasticsearch.client.indices.PutIndexTemplateRequest;
import org.jetlinks.community.elastic.search.ElasticRestClient;
import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesRequest;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
import org.jetlinks.community.elastic.search.index.ElasticSearchIndexMetadata;
import org.jetlinks.community.elastic.search.index.ElasticSearchIndexProperties;
import org.jetlinks.community.elastic.search.utils.ReactorActionListener;
import org.jetlinks.community.elastic.search.service.reactive.ReactiveElasticsearchClient;
import reactor.core.publisher.Mono;
import java.util.Collections;
@ -19,8 +15,8 @@ import java.util.Map;
public abstract class TemplateElasticSearchIndexStrategy extends AbstractElasticSearchIndexStrategy {
public TemplateElasticSearchIndexStrategy(String id, ElasticRestClient client, ElasticSearchIndexProperties properties) {
super(id, client,properties);
public TemplateElasticSearchIndexStrategy(String id, ReactiveElasticsearchClient client, ElasticSearchIndexProperties properties) {
super(id, client, properties);
}
protected String getTemplate(String index) {
@ -45,15 +41,12 @@ public abstract class TemplateElasticSearchIndexStrategy extends AbstractElastic
@Override
public Mono<Void> putIndex(ElasticSearchIndexMetadata metadata) {
return ReactorActionListener
.<AcknowledgedResponse>mono(listener -> client.getWriteClient()
.indices()//修改索引模版
.putTemplateAsync(createIndexTemplateRequest(metadata), RequestOptions.DEFAULT, listener))
return client
.updateTemplate(createIndexTemplateRequest(metadata))
//修改当前索引
.then(doPutIndex(metadata.newIndexName(getIndexForSave(metadata.getIndex())), true));
}
protected PutIndexTemplateRequest createIndexTemplateRequest(ElasticSearchIndexMetadata metadata) {
String index = wrapIndex(metadata.getIndex());
PutIndexTemplateRequest request = new PutIndexTemplateRequest(getTemplate(index));
@ -62,7 +55,7 @@ public abstract class TemplateElasticSearchIndexStrategy extends AbstractElastic
Map<String, Object> mappingConfig = new HashMap<>();
mappingConfig.put("properties", createElasticProperties(metadata.getProperties()));
mappingConfig.put("dynamic_templates", createDynamicTemplates());
request.mapping(mappingConfig);
request.mapping("_doc",mappingConfig);
request.patterns(getIndexPatterns(index));
return request;
}
@ -70,11 +63,7 @@ public abstract class TemplateElasticSearchIndexStrategy extends AbstractElastic
@Override
public Mono<ElasticSearchIndexMetadata> loadIndexMetadata(String index) {
return ReactorActionListener
.<GetIndexTemplatesResponse>mono(listener -> client.getQueryClient()
.indices()
.getIndexTemplateAsync(new GetIndexTemplatesRequest(getTemplate(index)), RequestOptions.DEFAULT, listener))
return client.getTemplate(new GetIndexTemplatesRequest(getTemplate(index)))
.filter(resp -> resp.getIndexTemplates().size() > 0)
.flatMap(resp -> Mono.justOrEmpty(convertMetadata(index, resp.getIndexTemplates().get(0).mappings())));
}

4
jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/strategies/TimeByMonthElasticSearchIndexStrategy.java
View File

@ -1,8 +1,8 @@
package org.jetlinks.community.elastic.search.index.strategies;
import org.hswebframework.utils.time.DateFormatter;
import org.jetlinks.community.elastic.search.ElasticRestClient;
import org.jetlinks.community.elastic.search.index.ElasticSearchIndexProperties;
import org.jetlinks.community.elastic.search.service.reactive.ReactiveElasticsearchClient;
import org.springframework.stereotype.Component;
import java.util.Date;
@ -18,7 +18,7 @@ public class TimeByMonthElasticSearchIndexStrategy extends TemplateElasticSearch
private final String format = "yyyy-MM";
public TimeByMonthElasticSearchIndexStrategy(ElasticRestClient client, ElasticSearchIndexProperties properties) {
public TimeByMonthElasticSearchIndexStrategy(ReactiveElasticsearchClient client, ElasticSearchIndexProperties properties) {
super("time-by-month", client,properties);
}

5
jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/DefaultElasticSearchService.java
View File

@ -59,9 +59,10 @@ import java.util.stream.Collectors;
* @author zhouhao
* @since 1.0
**/
@Service
//@Service
@Slf4j
@DependsOn("restHighLevelClient")
@Deprecated
public class DefaultElasticSearchService implements ElasticSearchService {
private final ElasticRestClient restClient;
@ -138,7 +139,7 @@ public class DefaultElasticSearchService implements ElasticSearchService {
convertQueryResult(tp2.getT1(), tp2.getT2(), mapper)
.collectList()
.filter(CollectionUtils::isNotEmpty)
.map(list -> PagerResult.of((int) tp2.getT2().getHits().getTotalHits(), list, queryParam))
.map(list -> PagerResult.of((int) tp2.getT2().getHits().getTotalHits().value, list, queryParam))
)
.switchIfEmpty(Mono.fromSupplier(PagerResult::empty));
}

1295
jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/reactive/DefaultReactiveElasticsearchClient.java Normal file
View File

File diff suppressed because it is too large Load Diff

59
jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/reactive/RawActionResponse.java Normal file
View File

@ -0,0 +1,59 @@
package org.jetlinks.community.elastic.search.service.reactive;
import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.springframework.http.HttpStatus;
import org.springframework.http.client.reactive.ClientHttpResponse;
import org.springframework.web.reactive.function.BodyExtractor;
import org.springframework.web.reactive.function.client.ClientResponse;
import java.io.IOException;
/**
* Extension to {@link ActionResponse} that also delegates to {@link ClientResponse}.
*
* @author Christoph Strobl
* @author Peter-Josef Meisch
* @author Mark Paluch
* @since 3.2
*/
class RawActionResponse extends ActionResponse {
private final ClientResponse delegate;
private RawActionResponse(ClientResponse delegate) {
this.delegate = delegate;
}
static RawActionResponse create(ClientResponse response) {
return new RawActionResponse(response);
}
public HttpStatus statusCode() {
return delegate.statusCode();
}
/*
* (non-Javadoc)
* @see org.springframework.web.reactive.function.client.ClientResponse#headers()
*/
public ClientResponse.Headers headers() {
return delegate.headers();
}
/*
* (non-Javadoc)
* @see org.springframework.web.reactive.function.client.ClientResponse#body(org.springframework.web.reactive.function.BodyExtractor)
*/
public <T> T body(BodyExtractor<T, ? super ClientHttpResponse> extractor) {
return delegate.body(extractor);
}
/*
* (non-Javadoc)
* until Elasticsearch 7.4 this empty implementation was available in the abstract base class
*/
@Override
public void writeTo(StreamOutput out) throws IOException {
}
}

199
jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/reactive/ReactiveAggregationService.java Normal file
View File

@ -0,0 +1,199 @@
package org.jetlinks.community.elastic.search.service.reactive;
import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.search.aggregations.bucket.histogram.ExtendedBounds;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.hswebframework.ezorm.core.param.QueryParam;
import org.hswebframework.ezorm.core.param.TermType;
import org.jetlinks.community.elastic.search.aggreation.bucket.Bucket;
import org.jetlinks.community.elastic.search.aggreation.bucket.BucketAggregationsStructure;
import org.jetlinks.community.elastic.search.aggreation.bucket.BucketResponse;
import org.jetlinks.community.elastic.search.aggreation.bucket.Sort;
import org.jetlinks.community.elastic.search.aggreation.enums.BucketType;
import org.jetlinks.community.elastic.search.aggreation.enums.MetricsType;
import org.jetlinks.community.elastic.search.aggreation.enums.OrderType;
import org.jetlinks.community.elastic.search.aggreation.metrics.MetricsAggregationStructure;
import org.jetlinks.community.elastic.search.index.ElasticSearchIndexManager;
import org.jetlinks.community.elastic.search.service.AggregationService;
import org.jetlinks.community.elastic.search.service.DefaultElasticSearchService;
import org.jetlinks.community.elastic.search.utils.ElasticSearchConverter;
import org.jetlinks.community.timeseries.query.AggregationQueryParam;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import java.util.*;
import java.util.stream.Collectors;
/**
* @author bsetfeng
* @since 1.0
**/
@Service
@Slf4j
public class ReactiveAggregationService implements AggregationService {
private final ReactiveElasticsearchClient restClient;
private final ElasticSearchIndexManager indexManager;
@Autowired
public ReactiveAggregationService(ElasticSearchIndexManager indexManager,
ReactiveElasticsearchClient restClient) {
this.restClient = restClient;
this.indexManager = indexManager;
}
private Mono<SearchSourceBuilder> createSearchSourceBuilder(QueryParam queryParam, String index) {
return indexManager
.getIndexMetadata(index)
.map(metadata -> ElasticSearchConverter.convertSearchSourceBuilder(queryParam, metadata));
}
@Override
public Flux<Map<String, Object>> aggregation(String[] index, AggregationQueryParam aggregationQueryParam) {
QueryParam queryParam = prepareQueryParam(aggregationQueryParam);
BucketAggregationsStructure structure = createAggParameter(aggregationQueryParam);
return Flux.fromArray(index)
.flatMap(idx -> Mono.zip(indexManager.getIndexStrategy(idx), Mono.just(idx)))
.collectList()
.flatMap(strategy ->
createSearchSourceBuilder(queryParam, index[0])
.map(builder ->
new SearchRequest(strategy
.stream()
.map(tp2 -> tp2.getT1().getIndexForSearch(tp2.getT2()))
.toArray(String[]::new))
.indicesOptions(DefaultElasticSearchService.indexOptions)
.source(builder.size(0).aggregation(structure.getType().aggregationBuilder(structure))
)
)
)
.flatMap(restClient::searchForPage)
.filter(response -> response.getAggregations() != null)
.map(response -> BucketResponse.builder()
.name(structure.getName())
.buckets(structure.getType().convert(response.getAggregations().get(structure.getName())))
.build())
.flatMapIterable(BucketsParser::convert)
.take(aggregationQueryParam.getLimit())
;
}
static class BucketsParser {
private final List<Map<String, Object>> result = new ArrayList<>();
public static List<Map<String, Object>> convert(BucketResponse response) {
return new BucketsParser(response).result;
}
public BucketsParser(BucketResponse response) {
this(response.getBuckets());
}
public BucketsParser(List<Bucket> buckets) {
buckets.forEach(bucket -> parser(bucket, new HashMap<>()));
}
public void parser(Bucket bucket, Map<String, Object> fMap) {
addBucketProperty(bucket, fMap);
if (bucket.getBuckets() != null && !bucket.getBuckets().isEmpty()) {
bucket.getBuckets().forEach(b -> {
Map<String, Object> map = new HashMap<>(fMap);
addBucketProperty(b, map);
parser(b, map);
});
} else {
result.add(fMap);
}
}
private void addBucketProperty(Bucket bucket, Map<String, Object> fMap) {
fMap.put(bucket.getName(), bucket.getKey());
fMap.putAll(bucket.toMap());
}
}
protected static QueryParam prepareQueryParam(AggregationQueryParam param) {
QueryParam queryParam = param.getQueryParam().clone();
queryParam.setPaging(false);
queryParam.and(param.getTimeProperty(), TermType.btw, Arrays.asList(calculateStartWithTime(param), param.getEndWithTime()));
if (queryParam.getSorts().isEmpty()) {
queryParam.orderBy(param.getTimeProperty()).desc();
}
return queryParam;
}
protected BucketAggregationsStructure createAggParameter(AggregationQueryParam param) {
List<BucketAggregationsStructure> structures = new ArrayList<>();
if (param.getGroupByTime() != null) {
structures.add(convertAggGroupTimeStructure(param));
}
if (param.getGroupBy() != null && !param.getGroupBy().isEmpty()) {
structures.addAll(getTermTypeStructures(param));
}
for (int i = 0, size = structures.size(); i < size; i++) {
if (i < size - 1) {
structures.get(i).setSubBucketAggregation(Collections.singletonList(structures.get(i + 1)));
}
if (i == size - 1) {
structures.get(i)
.setSubMetricsAggregation(param
.getAggColumns()
.stream()
.map(agg -> {
MetricsAggregationStructure metricsAggregationStructure = new MetricsAggregationStructure();
metricsAggregationStructure.setField(agg.getProperty());
metricsAggregationStructure.setName(agg.getAlias());
metricsAggregationStructure.setType(MetricsType.of(agg.getAggregation().name()));
return metricsAggregationStructure;
}).collect(Collectors.toList()));
}
}
return structures.get(0);
}
protected BucketAggregationsStructure convertAggGroupTimeStructure(AggregationQueryParam param) {
BucketAggregationsStructure structure = new BucketAggregationsStructure();
structure.setInterval(param.getGroupByTime().getInterval().toString());
structure.setType(BucketType.DATE_HISTOGRAM);
structure.setFormat(param.getGroupByTime().getFormat());
structure.setName(param.getGroupByTime().getAlias());
structure.setField(param.getGroupByTime().getProperty());
structure.setSort(Sort.desc(OrderType.KEY));
structure.setExtendedBounds(getExtendedBounds(param));
return structure;
}
protected static ExtendedBounds getExtendedBounds(AggregationQueryParam param) {
return new ExtendedBounds(calculateStartWithTime(param), param.getEndWithTime());
}
private static long calculateStartWithTime(AggregationQueryParam param) {
long startWithParam = param.getStartWithTime();
// if (param.getGroupByTime() != null && param.getGroupByTime().getInterval() != null) {
// long timeInterval = param.getGroupByTime().getInterval().toMillis() * param.getLimit();
// long tempStartWithParam = param.getEndWithTime() - timeInterval;
// startWithParam = Math.max(tempStartWithParam, startWithParam);
// }
return startWithParam;
}
protected List<BucketAggregationsStructure> getTermTypeStructures(AggregationQueryParam param) {
return param.getGroupBy()
.stream()
.map(group -> {
BucketAggregationsStructure structure = new BucketAggregationsStructure();
structure.setType(BucketType.TERMS);
structure.setSize(param.getLimit());
structure.setField(group.getProperty());
structure.setName(group.getAlias());
return structure;
}).collect(Collectors.toList());
}
}

413
jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/reactive/ReactiveElasticSearchService.java Normal file
View File

@ -0,0 +1,413 @@
package org.jetlinks.community.elastic.search.service.reactive;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.MultiSearchRequest;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.client.core.CountRequest;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.reindex.BulkByScrollResponse;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.hswebframework.ezorm.core.param.QueryParam;
import org.hswebframework.utils.time.DateFormatter;
import org.hswebframework.utils.time.DefaultDateFormatter;
import org.hswebframework.web.api.crud.entity.PagerResult;
import org.hswebframework.web.bean.FastBeanCopier;
import org.jetlinks.core.utils.FluxUtils;
import org.jetlinks.community.elastic.search.index.ElasticSearchIndexManager;
import org.jetlinks.community.elastic.search.index.ElasticSearchIndexMetadata;
import org.jetlinks.community.elastic.search.service.ElasticSearchService;
import org.jetlinks.community.elastic.search.utils.ElasticSearchConverter;
import org.jetlinks.community.elastic.search.utils.QueryParamTranslator;
import org.reactivestreams.Publisher;
import org.springframework.context.annotation.DependsOn;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import reactor.core.publisher.BufferOverflowStrategy;
import reactor.core.publisher.Flux;
import reactor.core.publisher.FluxSink;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
import reactor.util.function.Tuple2;
import reactor.util.function.Tuples;
import javax.annotation.PreDestroy;
import java.time.Duration;
import java.util.*;
import java.util.function.Function;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
/**
* @author zhouhao
* @since 1.0
**/
@Service("elasticSearchService")
@Slf4j
@DependsOn("reactiveElasticsearchClient")
public class ReactiveElasticSearchService implements ElasticSearchService {
private final ReactiveElasticsearchClient restClient;
private final ElasticSearchIndexManager indexManager;
FluxSink<Buffer> sink;
public static final IndicesOptions indexOptions = IndicesOptions.fromOptions(
true, true, false, false
);
static {
DateFormatter.supportFormatter.add(new DefaultDateFormatter(Pattern.compile("[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.+"), "yyyy-MM-dd'T'HH:mm:ss.SSSZ"));
}
public ReactiveElasticSearchService(ReactiveElasticsearchClient restClient,
ElasticSearchIndexManager indexManager) {
this.restClient = restClient;
init();
this.indexManager = indexManager;
}
@Override
public <T> Flux<T> multiQuery(String[] index, Collection<QueryParam> queryParam, Function<Map<String, Object>, T> mapper) {
return indexManager
.getIndexesMetadata(index)
.flatMap(idx -> Mono.zip(
Mono.just(idx), getIndexForSearch(idx.getIndex())
))
.take(1)
.singleOrEmpty()
.flatMapMany(indexMetadata -> {
MultiSearchRequest request = new MultiSearchRequest();
return Flux
.fromIterable(queryParam)
.flatMap(entry -> createSearchRequest(entry, index))
.doOnNext(request::add)
.then(Mono.just(request))
.flatMapMany(searchRequest -> restClient.multiSearch(searchRequest)
.flatMapMany(response -> Flux.fromArray(response.getResponses()))
.flatMap(item -> {
if (item.isFailure()) {
log.warn(item.getFailureMessage(), item.getFailure());
return Mono.empty();
}
return Flux.fromIterable(translate((map) -> mapper.apply(indexMetadata.getT1().convertFromElastic(map)), item.getResponse()));
}))
;
});
}
public <T> Flux<T> query(String index, QueryParam queryParam, Function<Map<String, Object>, T> mapper) {
return this
.doQuery(new String[]{index}, queryParam)
.flatMapMany(tp2 -> convertQueryResult(tp2.getT1(), tp2.getT2(), mapper));
}
public <T> Flux<T> query(String[] index, QueryParam queryParam, Function<Map<String, Object>, T> mapper) {
return this
.doQuery(index, queryParam)
.flatMapMany(tp2 -> convertQueryResult(tp2.getT1(), tp2.getT2(), mapper));
}
@Override
public <T> Mono<PagerResult<T>> queryPager(String[] index, QueryParam queryParam, Function<Map<String, Object>, T> mapper) {
return this.doQuery(index, queryParam)
.flatMap(tp2 ->
convertQueryResult(tp2.getT1(), tp2.getT2(), mapper)
.collectList()
.filter(CollectionUtils::isNotEmpty)
.map(list -> PagerResult.of((int) tp2.getT2().getHits().getTotalHits().value, list, queryParam))
)
.switchIfEmpty(Mono.fromSupplier(PagerResult::empty));
}
private <T> Flux<T> convertQueryResult(List<ElasticSearchIndexMetadata> indexList,
SearchResponse response,
Function<Map<String, Object>, T> mapper) {
Map<String, ElasticSearchIndexMetadata> metadata = indexList
.stream()
.collect(Collectors.toMap(ElasticSearchIndexMetadata::getIndex, Function.identity()));
return Flux
.fromIterable(response.getHits())
.map(hit -> {
Map<String, Object> hitMap = hit.getSourceAsMap();
if (StringUtils.isEmpty(hitMap.get("id"))) {
hitMap.put("id", hit.getId());
}
return mapper
.apply(Optional
.ofNullable(metadata.get(hit.getIndex())).orElse(indexList.get(0))
.convertFromElastic(hitMap));
});
}
private Mono<Tuple2<List<ElasticSearchIndexMetadata>, SearchResponse>> doQuery(String[] index,
QueryParam queryParam) {
return indexManager
.getIndexesMetadata(index)
.collectList()
.filter(CollectionUtils::isNotEmpty)
.flatMap(metadataList -> this
.createSearchRequest(queryParam, metadataList)
.flatMap(restClient::searchForPage)
.map(response -> Tuples.of(metadataList, response))
).onErrorResume(err -> {
log.error(err.getMessage(), err);
return Mono.empty();
});
}
@Override
public Mono<Long> count(String[] index, QueryParam queryParam) {
QueryParam param = queryParam.clone();
param.setPaging(false);
return createSearchRequest(param, index)
.flatMap(this::doCount)
.defaultIfEmpty(0L)
.onErrorReturn(err -> {
log.error("query elastic error", err);
return true;
}, 0L);
}
@Override
public Mono<Long> delete(String index, QueryParam queryParam) {
return createQueryBuilder(queryParam, index)
.flatMap(request -> restClient.deleteBy(delete -> delete.setQuery(request).indices(index)))
.map(BulkByScrollResponse::getDeleted);
}
@Override
public <T> Mono<Void> commit(String index, T payload) {
sink.next(new Buffer(index, payload));
return Mono.empty();
}
@Override
public <T> Mono<Void> commit(String index, Collection<T> payload) {
for (T t : payload) {
sink.next(new Buffer(index, t));
}
return Mono.empty();
}
@Override
public <T> Mono<Void> commit(String index, Publisher<T> data) {
return Flux.from(data)
.flatMap(d -> commit(index, d))
.then();
}
@Override
public <T> Mono<Void> save(String index, T payload) {
return save(index, Mono.just(payload));
}
@Override
public <T> Mono<Void> save(String index, Publisher<T> data) {
return Flux.from(data)
.map(v -> new Buffer(index, v))
.collectList()
.flatMap(this::doSave)
.then();
}
@Override
public <T> Mono<Void> save(String index, Collection<T> payload) {
return save(index, Flux.fromIterable(payload));
}
@PreDestroy
public void shutdown() {
sink.complete();
}
//@PostConstruct
public void init() {
//最小间隔
int flushRate = Integer.getInteger("elasticsearch.buffer.rate", 1000);
//缓冲最大数量
int bufferSize = Integer.getInteger("elasticsearch.buffer.size", 3000);
//缓冲超时时间
Duration bufferTimeout = Duration.ofSeconds(Integer.getInteger("elasticsearch.buffer.timeout", 3));
//缓冲背压
int bufferBackpressure = Integer.getInteger("elasticsearch.buffer.backpressure", 64);
FluxUtils.bufferRate(
Flux.<Buffer>create(sink -> this.sink = sink),
flushRate,
bufferSize,
bufferTimeout)
.onBackpressureBuffer(bufferBackpressure,
drop -> System.err.println("无法处理更多索引请求!"),
BufferOverflowStrategy.DROP_OLDEST)
.parallel()
.runOn(Schedulers.parallel())
.flatMap(buffers -> {
long time = System.currentTimeMillis();
return this
.doSave(buffers)
.doOnNext((len) -> log.trace("保存ElasticSearch数据成功,数量:{},耗时:{}ms", len, (System.currentTimeMillis() - time)))
.onErrorContinue((err, obj) -> {
//这里的错误都输出到控制台,输入到slf4j可能会造成日志递归.
System.err.println("保存ElasticSearch数据失败:\n" + org.hswebframework.utils.StringUtils.throwable2String(err));
});
})
.subscribe();
}
@AllArgsConstructor
@Getter
static class Buffer {
String index;
Object payload;
}
private Mono<String> getIndexForSave(String index) {
return indexManager
.getIndexStrategy(index)
.map(strategy -> strategy.getIndexForSave(index));
}
private Mono<String> getIndexForSearch(String index) {
return indexManager
.getIndexStrategy(index)
.map(strategy -> strategy.getIndexForSearch(index));
}
protected Mono<Integer> doSave(Collection<Buffer> buffers) {
return Flux.fromIterable(buffers)
.groupBy(Buffer::getIndex)
.flatMap(group -> {
String index = group.key();
return this.getIndexForSave(index)
.zipWith(indexManager.getIndexMetadata(index))
.flatMapMany(tp2 ->
group.map(buffer -> {
Map<String, Object> data = FastBeanCopier.copy(buffer.getPayload(), HashMap::new);
IndexRequest request;
if (data.get("id") != null) {
request = new IndexRequest(tp2.getT1()).type("_doc").id(String.valueOf(data.get("id")));
} else {
request = new IndexRequest(tp2.getT1()).type("_doc");
}
request.source(tp2.getT2().convertToElastic(data));
return request;
}));
})
.collectList()
.filter(CollectionUtils::isNotEmpty)
.flatMap(lst -> {
BulkRequest request = new BulkRequest();
lst.forEach(request::add);
return restClient.bulk(request);
})
.thenReturn(buffers.size());
}
@SneakyThrows
protected void checkResponse(BulkResponse response) {
if (response.hasFailures()) {
for (BulkItemResponse item : response.getItems()) {
if (item.isFailed()) {
throw item.getFailure().getCause();
}
}
}
}
private <T> List<T> translate(Function<Map<String, Object>, T> mapper, SearchResponse response) {
return Arrays.stream(response.getHits().getHits())
.map(hit -> {
Map<String, Object> hitMap = hit.getSourceAsMap();
if (StringUtils.isEmpty(hitMap.get("id"))) {
hitMap.put("id", hit.getId());
}
return mapper.apply(hitMap);
})
.collect(Collectors.toList());
}
private Flux<SearchHit> doSearch(SearchRequest request) {
return restClient
.search(request)
.onErrorResume(err -> {
log.error("query elastic error", err);
return Mono.empty();
});
}
private Mono<Long> doCount(SearchRequest request) {
return restClient
.count(request)
.onErrorResume(err -> {
log.error("query elastic error", err);
return Mono.empty();
});
}
protected Mono<SearchRequest> createSearchRequest(QueryParam queryParam, String... indexes) {
return indexManager
.getIndexesMetadata(indexes)
.collectList()
.filter(CollectionUtils::isNotEmpty)
.flatMap(list -> createSearchRequest(queryParam, list));
}
protected Mono<SearchRequest> createSearchRequest(QueryParam queryParam, List<ElasticSearchIndexMetadata> indexes) {
SearchSourceBuilder builder = ElasticSearchConverter.convertSearchSourceBuilder(queryParam, indexes.get(0));
return Flux.fromIterable(indexes)
.flatMap(index -> getIndexForSearch(index.getIndex()))
.collectList()
.map(indexList ->
new SearchRequest(indexList.toArray(new String[0]))
.source(builder)
.indicesOptions(indexOptions));
}
protected Mono<QueryBuilder> createQueryBuilder(QueryParam queryParam, String index) {
return indexManager
.getIndexMetadata(index)
.map(metadata -> QueryParamTranslator.createQueryBuilder(queryParam, metadata))
.switchIfEmpty(Mono.fromSupplier(() -> QueryParamTranslator.createQueryBuilder(queryParam, null)));
}
protected Mono<CountRequest> createCountRequest(QueryParam queryParam, List<ElasticSearchIndexMetadata> indexes) {
QueryParam tempQueryParam = queryParam.clone();
tempQueryParam.setPaging(false);
tempQueryParam.setSorts(Collections.emptyList());
SearchSourceBuilder builder = ElasticSearchConverter.convertSearchSourceBuilder(queryParam, indexes.get(0));
return Flux.fromIterable(indexes)
.flatMap(index -> getIndexForSearch(index.getIndex()))
.collectList()
.map(indexList -> new CountRequest(indexList.toArray(new String[0])).source(builder));
}
private Mono<CountRequest> createCountRequest(QueryParam queryParam, String... index) {
return indexManager
.getIndexesMetadata(index)
.collectList()
.filter(CollectionUtils::isNotEmpty)
.flatMap(list -> createCountRequest(queryParam, list));
}
}

29
jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/reactive/ReactiveElasticsearchClient.java Normal file
View File

@ -0,0 +1,29 @@
package org.jetlinks.community.elastic.search.service.reactive;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesRequest;
import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
import org.elasticsearch.action.search.MultiSearchRequest;
import org.elasticsearch.action.search.MultiSearchResponse;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import reactor.core.publisher.Mono;
public interface ReactiveElasticsearchClient extends
org.springframework.data.elasticsearch.client.reactive.ReactiveElasticsearchClient
, org.springframework.data.elasticsearch.client.reactive.ReactiveElasticsearchClient.Indices {
Mono<SearchResponse> searchForPage(SearchRequest request);
Mono<MultiSearchResponse> multiSearch(MultiSearchRequest request);
Mono<GetMappingsResponse> getMapping(GetMappingsRequest request);
Mono<GetIndexTemplatesResponse> getTemplate(GetIndexTemplatesRequest request);
Mono<AcknowledgedResponse> updateTemplate(PutIndexTemplateRequest request);
}

8
jetlinks-standalone/src/main/resources/application-embedded.yml
View File

@ -14,6 +14,14 @@ spring:
password:
pool:
max-size: 32
data:
elasticsearch:
client:
reactive:
endpoints: ${elasticsearch.client.host}:${elasticsearch.client.port}
max-in-memory-size: 100MB
socket-timeout: ${elasticsearch.client.socket-timeout}
connection-timeout: ${elasticsearch.client.socket-timeout}
easyorm:
default-schema: PUBLIC # 数据库默认的schema
dialect: h2 #数据库方言

2
pom.xml
View File

@ -26,7 +26,7 @@
<r2dbc.version>Arabba-SR6</r2dbc.version>
<vertx.version>3.8.5</vertx.version>
<netty.version>4.1.50.Final</netty.version>
<elasticsearch.version>6.8.11</elasticsearch.version>
<elasticsearch.version>7.9.0</elasticsearch.version>
<reactor.excel.version>1.0.0</reactor.excel.version>
<reactor.ql.version>1.0.6</reactor.ql.version>
<fastjson.version>1.2.70</fastjson.version>