From 6c082000fabb3e86c25bcec9d117f42064c0cd27 Mon Sep 17 00:00:00 2001 From: zhouhao Date: Thu, 5 Mar 2020 17:27:29 +0800 Subject: [PATCH] =?UTF-8?q?=E9=87=8D=E6=9E=84es=EF=BC=8C=E4=BC=98=E5=8C=96?= =?UTF-8?q?=E7=B4=A2=E5=BC=95=E7=AE=A1=E7=90=86=EF=BC=8C=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E6=8C=89=E6=97=B6=E9=97=B4=E5=88=86=E8=A1=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../aggreation/DefaultAggregationService.java | 226 +++++++++++++--- .../search/aggreation/enums/BucketType.java | 3 - .../ElasticSearchConfiguration.java | 12 +- ...DateFormat.java => ElasticDateFormat.java} | 11 +- .../search/enums/ElasticPropertyType.java | 66 +++++ .../elastic/search/enums/FieldType.java | 57 ----- .../search/enums/IndexPatternEnum.java | 18 -- .../search/enums/IndexStrategyEnum.java | 17 -- .../elastic/search/index/CreateIndex.java | 68 ----- .../DefaultElasticSearchIndexManager.java | 62 +++++ .../DefaultElasticSearchIndexMetadata.java | 53 ++++ .../index/DefaultIndexOperationService.java | 136 ---------- .../elastic/search/index/ElasticIndex.java | 28 -- .../index/ElasticSearchIndexManager.java | 13 + .../index/ElasticSearchIndexMetadata.java | 28 ++ .../index/ElasticSearchIndexStrategy.java | 45 ++++ .../DefaultIndexAliasOperationService.java | 66 ----- .../index/mapping/IndexMappingMetadata.java | 6 +- .../index/mapping/IndicesMappingCenter.java | 26 -- .../search/index/mapping/MappingFactory.java | 77 ------ .../index/mapping/SingleMappingMetadata.java | 8 +- .../search/index/setting/SettingFactory.java | 40 --- .../AbstractElasticSearchIndexStrategy.java | 187 ++++++++++++++ .../DirectElasticSearchIndexStrategy.java | 35 +++ .../TemplateElasticSearchIndexStrategy.java | 71 ++++++ ...TimeByMonthElasticSearchIndexStrategy.java | 28 ++ .../DefaultIndexTemplateOperationService.java | 67 ----- .../manager/DefaultIndexStrategyProvider.java | 50 ---- .../elastic/search/manager/IndexManager.java | 20 -- .../search/manager/IndexPatternManager.java | 12 - .../search/manager/IndexStrategyManager.java | 12 - .../search/manager/IndexStrategyProvider.java | 37 --- .../search/manager/StandardsIndexInit.java | 21 -- .../search/manager/StandardsIndexManager.java | 21 -- .../manager/StandardsIndexManagerCenter.java | 124 --------- .../search/manager/entity/IndexStrategy.java | 19 -- .../manager/strategy/IndexDayPattern.java | 26 -- .../manager/strategy/IndexMonthPattern.java | 27 -- .../manager/strategy/IndexSuffixStrategy.java | 24 -- .../AbstractQueryParamTranslateService.java | 35 --- .../search/parser/DefaultLinkTypeParser.java | 14 +- .../DefaultQueryParamTranslateService.java | 52 ---- .../parser/QueryParamTranslateService.java | 14 - .../search/service/AggregationService.java | 14 +- .../service/DefaultElasticSearchService.java | 207 +++++++-------- .../search/service/ElasticSearchService.java | 55 +++- .../service/IndexAliasOperationService.java | 15 -- .../search/service/IndexOperationService.java | 18 -- .../IndexTemplateOperationService.java | 15 -- .../timeseries/AbstractTimeSeriesService.java | 20 -- .../ESAbstractTimeSeriesManager.java | 77 ------ .../search/timeseries/ESAggregationData.java | 24 -- .../timeseries/ESTimeSeriesManager.java | 100 -------- .../timeseries/ESTimeSeriesService.java | 241 ------------------ .../ElasticSearchTimeSeriesManager.java | 69 +++++ .../ElasticSearchTimeSeriesService.java | 68 +++++ .../search/timeseries/IndexAliasProvider.java | 14 - .../TimeSeriesServiceRegisterCenter.java | 32 --- .../translate/QueryParamTranslator.java | 44 ---- .../elastic/search/utils/DateTimeUtils.java | 54 ---- .../search/utils/ElasticSearchConverter.java | 63 +++++ .../search/utils/QueryParamTranslator.java | 81 ++++++ .../search/utils/ReactorActionListener.java | 56 ++++ .../elastic/search/utils/TermCommonUtils.java | 2 +- .../configuration/LoggingConfiguration.java | 8 - .../handler/AccessLoggerEventHandler.java | 42 +-- .../event/handler/LoggerIndexProvider.java | 6 +- .../handler/SystemLoggerEventHandler.java | 35 ++- .../RuleEngineLogIndexInitialize.java | 57 ++--- .../engine/event/handler/RuleLogHandler.java | 5 +- .../timeseries/TimeSeriesManager.java | 2 + .../timeseries/query/AggregationData.java | 3 + .../query/AggregationQueryParam.java | 5 +- .../community/timeseries/query/TimeGroup.java | 8 +- .../src/main/resources/application.yml | 2 + 75 files changed, 1383 insertions(+), 2021 deletions(-) rename jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/enums/{FieldDateFormat.java => ElasticDateFormat.java} (80%) create mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticPropertyType.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/enums/FieldType.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/enums/IndexPatternEnum.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/enums/IndexStrategyEnum.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/CreateIndex.java create mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/DefaultElasticSearchIndexManager.java create mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/DefaultElasticSearchIndexMetadata.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/DefaultIndexOperationService.java create mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexManager.java create mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexMetadata.java create mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexStrategy.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/alias/DefaultIndexAliasOperationService.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/mapping/IndicesMappingCenter.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/mapping/MappingFactory.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/setting/SettingFactory.java create mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/strategies/AbstractElasticSearchIndexStrategy.java create mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/strategies/DirectElasticSearchIndexStrategy.java create mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/strategies/TemplateElasticSearchIndexStrategy.java create mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/strategies/TimeByMonthElasticSearchIndexStrategy.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/template/DefaultIndexTemplateOperationService.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/DefaultIndexStrategyProvider.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/IndexManager.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/IndexPatternManager.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/IndexStrategyManager.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/IndexStrategyProvider.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/StandardsIndexInit.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/StandardsIndexManager.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/StandardsIndexManagerCenter.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/entity/IndexStrategy.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/strategy/IndexDayPattern.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/strategy/IndexMonthPattern.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/strategy/IndexSuffixStrategy.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/parser/AbstractQueryParamTranslateService.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/parser/DefaultQueryParamTranslateService.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/parser/QueryParamTranslateService.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/IndexAliasOperationService.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/IndexOperationService.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/IndexTemplateOperationService.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/timeseries/AbstractTimeSeriesService.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/timeseries/ESAbstractTimeSeriesManager.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/timeseries/ESAggregationData.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/timeseries/ESTimeSeriesManager.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/timeseries/ESTimeSeriesService.java create mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/timeseries/ElasticSearchTimeSeriesManager.java create mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/timeseries/ElasticSearchTimeSeriesService.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/timeseries/IndexAliasProvider.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/timeseries/TimeSeriesServiceRegisterCenter.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/translate/QueryParamTranslator.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/utils/DateTimeUtils.java create mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/utils/ElasticSearchConverter.java create mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/utils/QueryParamTranslator.java create mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/utils/ReactorActionListener.java diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/DefaultAggregationService.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/DefaultAggregationService.java index dfa1beba..40880968 100644 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/DefaultAggregationService.java +++ b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/DefaultAggregationService.java @@ -8,22 +8,35 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.search.aggregations.bucket.histogram.ExtendedBounds; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.hswebframework.ezorm.core.param.QueryParam; +import org.hswebframework.ezorm.core.param.TermType; import org.jetlinks.community.elastic.search.ElasticRestClient; +import org.jetlinks.community.elastic.search.aggreation.bucket.Bucket; import org.jetlinks.community.elastic.search.aggreation.bucket.BucketAggregationsStructure; import org.jetlinks.community.elastic.search.aggreation.bucket.BucketResponse; +import org.jetlinks.community.elastic.search.aggreation.bucket.Sort; +import org.jetlinks.community.elastic.search.aggreation.enums.BucketType; +import org.jetlinks.community.elastic.search.aggreation.enums.MetricsType; +import org.jetlinks.community.elastic.search.aggreation.enums.OrderType; import org.jetlinks.community.elastic.search.aggreation.metrics.MetricsAggregationStructure; import org.jetlinks.community.elastic.search.aggreation.metrics.MetricsResponse; -import org.jetlinks.community.elastic.search.index.ElasticIndex; -import org.jetlinks.community.elastic.search.parser.QueryParamTranslateService; +import org.jetlinks.community.elastic.search.index.ElasticSearchIndexManager; import org.jetlinks.community.elastic.search.service.AggregationService; -import org.jetlinks.community.elastic.search.service.IndexOperationService; +import org.jetlinks.community.elastic.search.utils.ElasticSearchConverter; +import org.jetlinks.community.elastic.search.utils.ReactorActionListener; +import org.jetlinks.community.timeseries.query.AggregationQueryParam; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.core.publisher.MonoSink; +import java.time.Duration; +import java.util.*; +import java.util.stream.Collectors; + /** * @author bsetfeng * @since 1.0 @@ -32,28 +45,22 @@ import reactor.core.publisher.MonoSink; @Slf4j public class DefaultAggregationService implements AggregationService { - private final QueryParamTranslateService translateService; - private final ElasticRestClient restClient; - private final IndexOperationService indexOperationService; + private final ElasticSearchIndexManager indexManager; @Autowired - public DefaultAggregationService(IndexOperationService indexOperationService, - ElasticRestClient restClient, - QueryParamTranslateService translateService) { - this.indexOperationService = indexOperationService; + public DefaultAggregationService(ElasticSearchIndexManager indexManager, + ElasticRestClient restClient) { this.restClient = restClient; - this.translateService = translateService; + this.indexManager = indexManager; } - @Override - public Mono metricsAggregation(QueryParam queryParam, - MetricsAggregationStructure structure, - ElasticIndex provider) { - return searchSourceBuilderMono(queryParam, provider) - .map(builder -> new SearchRequest(provider.getStandardIndex()) + public Mono metricsAggregation(String index, QueryParam queryParam, + MetricsAggregationStructure structure) { + return createSearchSourceBuilder(queryParam, index) + .map(builder -> new SearchRequest(index) .source(builder.aggregation(structure.getType().aggregationBuilder(structure.getName(), structure.getField())))) .flatMap(request -> Mono.create(monoSink -> restClient.getQueryClient().searchAsync(request, RequestOptions.DEFAULT, translatorActionListener(monoSink)))) @@ -61,12 +68,15 @@ public class DefaultAggregationService implements AggregationService { } @Override - public Mono bucketAggregation(QueryParam queryParam, BucketAggregationsStructure structure, ElasticIndex provider) { - return searchSourceBuilderMono(queryParam, provider) - .map(builder -> new SearchRequest(provider.getStandardIndex()) + public Mono bucketAggregation(String index, QueryParam queryParam, BucketAggregationsStructure structure) { + return createSearchSourceBuilder(queryParam, index) + .map(builder -> new SearchRequest(index) .source(builder.aggregation(structure.getType().aggregationBuilder(structure)))) - .doOnNext(searchRequest -> - log.debug("聚合查询index:{},参数:{}", provider.getStandardIndex(), JSON.toJSON(searchRequest.source().toString()))) + .doOnNext(searchRequest -> { + if (log.isDebugEnabled()) { + log.debug("聚合查询ElasticSearch:{},参数:{}", index, JSON.toJSON(searchRequest.source().toString())); + } + }) .flatMap(request -> Mono.create(monoSink -> restClient .getQueryClient() @@ -79,13 +89,11 @@ public class DefaultAggregationService implements AggregationService { } - private Mono searchSourceBuilderMono(QueryParam queryParam, ElasticIndex provider) { - QueryParam tempQueryParam = queryParam.clone(); - tempQueryParam.setPaging(false); - return indexOperationService.getIndexMappingMetadata(provider.getStandardIndex()) - .map(metadata -> translateService.translate(tempQueryParam, metadata)) - .doOnError(e -> log.error("解析queryParam错误, index:{}", provider.getStandardIndex(), e)); - // return Mono.just(translateService.translate(queryParam, IndexMappingMetadata.getInstance(provider.getStandardIndex()))); + private Mono createSearchSourceBuilder(QueryParam queryParam, String index) { + + return indexManager.getIndexMetadata(index) + .map(metadata -> ElasticSearchConverter.convertSearchSourceBuilder(queryParam, metadata)) + .doOnError(e -> log.error("解析queryParam错误:{}", index, e)); } private ActionListener translatorActionListener(MonoSink sink) { @@ -109,4 +117,164 @@ public class DefaultAggregationService implements AggregationService { } }; } + + @Override + public Flux> aggregation(String index, AggregationQueryParam aggregationQueryParam) { + QueryParam queryParam = prepareQueryParam(aggregationQueryParam); + BucketAggregationsStructure structure = createAggParameter(aggregationQueryParam); + return indexManager + .getIndexStrategy(index) + .flatMap(strategy -> + createSearchSourceBuilder(queryParam, index) + .map(builder -> + new SearchRequest(strategy.getIndexForSearch(index)) + .source(builder.aggregation(structure.getType().aggregationBuilder(structure))))) + .doOnNext(searchRequest -> { + if (log.isDebugEnabled()) { + log.debug("聚合查询ElasticSearch:{},参数:{}", index, JSON.toJSON(searchRequest.source().toString())); + } + }) + .flatMap(searchRequest -> + ReactorActionListener + .mono(listener -> + restClient.getQueryClient() + .searchAsync(searchRequest, RequestOptions.DEFAULT, listener) + )) + .map(response -> BucketResponse.builder() + .name(structure.getName()) + .buckets(structure.getType().convert(response.getAggregations().get(structure.getName()))) + .build()) + .flatMapIterable(BucketsParser::convert) + .take(aggregationQueryParam.getLimit()) + ; + } + + static class BucketsParser { + + private List> result = new ArrayList<>(); + + public static List> convert(BucketResponse response) { + return new BucketsParser(response).result; + } + + public BucketsParser(BucketResponse response) { + this(response.getBuckets()); + } + + public BucketsParser(List buckets) { + buckets.forEach(bucket -> parser(bucket, new HashMap<>())); + } + + public void parser(Bucket bucket, Map fMap) { + addBucketProperty(bucket, fMap); + if (bucket.getBuckets() != null && !bucket.getBuckets().isEmpty()) { + bucket.getBuckets().forEach(b -> { + Map map = new HashMap<>(fMap); + addBucketProperty(b, map); + parser(b, map); + }); + } else { + result.add(fMap); + } + } + + private void addBucketProperty(Bucket bucket, Map fMap) { + fMap.put(bucket.getName(), bucket.getKey()); + fMap.putAll(bucket.toMap()); + } + } + + protected static QueryParam prepareQueryParam(AggregationQueryParam param) { + QueryParam queryParam = param.getQueryParam().clone(); + queryParam.setPaging(false); + queryParam.and(param.getTimeProperty(), TermType.btw, Arrays.asList(calculateStartWithTime(param), param.getEndWithTime())); + if (queryParam.getSorts().isEmpty()) { + queryParam.orderBy(param.getTimeProperty()).desc(); + } + return queryParam; + } + + protected BucketAggregationsStructure createAggParameter(AggregationQueryParam param) { + List structures = new ArrayList<>(); + if (param.getGroupByTime() != null) { + structures.add(convertAggGroupTimeStructure(param)); + } + if (param.getGroupBy() != null && !param.getGroupBy().isEmpty()) { + structures.addAll(getTermTypeStructures(param)); + } + for (int i = 0, size = structures.size(); i < size; i++) { + if (i < size - 1) { + structures.get(i).setSubBucketAggregation(Collections.singletonList(structures.get(i + 1))); + } + if (i == size - 1) { + structures.get(i) + .setSubMetricsAggregation(param + .getAggColumns() + .stream() + .map(agg -> { + MetricsAggregationStructure metricsAggregationStructure = new MetricsAggregationStructure(); + metricsAggregationStructure.setField(agg.getProperty()); + metricsAggregationStructure.setName(agg.getAlias()); + metricsAggregationStructure.setType(MetricsType.of(agg.getAggregation().name())); + return metricsAggregationStructure; + }).collect(Collectors.toList())); + } + } + return structures.get(0); + } + + protected BucketAggregationsStructure convertAggGroupTimeStructure(AggregationQueryParam param) { + BucketAggregationsStructure structure = new BucketAggregationsStructure(); + structure.setInterval(durationFormat(param.getGroupByTime().getInterval())); + structure.setType(BucketType.DATE_HISTOGRAM); + structure.setFormat(param.getGroupByTime().getFormat()); + structure.setName(param.getGroupByTime().getAlias()); + structure.setField(param.getGroupByTime().getProperty()); + structure.setSort(Sort.desc(OrderType.KEY)); + structure.setExtendedBounds(getExtendedBounds(param)); + return structure; + } + + protected static ExtendedBounds getExtendedBounds(AggregationQueryParam param) { + return new ExtendedBounds(calculateStartWithTime(param), param.getEndWithTime()); + } + + private static long calculateStartWithTime(AggregationQueryParam param) { + long startWithParam = param.getStartWithTime(); + if (param.getGroupByTime() != null && param.getGroupByTime().getInterval() != null) { + long timeInterval = param.getGroupByTime().getInterval().toMillis() * param.getLimit(); + long tempStartWithParam = param.getEndWithTime() - timeInterval; + startWithParam = Math.max(tempStartWithParam, startWithParam); + } + return startWithParam; + } + + protected List getTermTypeStructures(AggregationQueryParam param) { + return param.getGroupBy() + .stream() + .map(group -> { + BucketAggregationsStructure structure = new BucketAggregationsStructure(); + structure.setType(BucketType.TERMS); + structure.setSize(param.getLimit()); + structure.setField(group.getProperty()); + structure.setName(group.getAlias()); + return structure; + }).collect(Collectors.toList()); + } + + protected static String durationFormat(Duration duration) { + String durationStr = duration.toString(); + if (durationStr.contains("S")) { + return duration.toMillis() / 1000 + "s"; + } else if (!durationStr.contains("S") && durationStr.contains("M")) { + return duration.toMinutes() + "m"; + } else if (!durationStr.contains("S") && !durationStr.contains("M")) { + if (duration.toHours() % 24 == 0) { + return duration.toDays() + "d"; + } else { + return duration.toHours() + "h"; + } + } + throw new UnsupportedOperationException("不支持的时间周期:" + duration.toString()); + } } diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/enums/BucketType.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/enums/BucketType.java index 6ea32ccd..63eafb21 100644 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/enums/BucketType.java +++ b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/enums/BucketType.java @@ -188,7 +188,4 @@ public enum BucketType { mapping.put(OrderBuilder.of("desc", OrderType.KEY), BucketOrder.key(false)); } - public static void main(String[] args) { - System.out.println(mapping.get(OrderBuilder.of("desc", OrderType.KEY)).toString()); - } } diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/configuration/ElasticSearchConfiguration.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/configuration/ElasticSearchConfiguration.java index fbcc1210..3993cf81 100644 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/configuration/ElasticSearchConfiguration.java +++ b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/configuration/ElasticSearchConfiguration.java @@ -21,15 +21,11 @@ public class ElasticSearchConfiguration { @Autowired private ElasticSearchProperties properties; - @Bean public ElasticRestClient elasticRestClient() { - RestHighLevelClient queryClient = new RestHighLevelClient(RestClient.builder(properties.createHosts()) - .setRequestConfigCallback(properties::applyRequestConfigBuilder) - .setHttpClientConfigCallback(properties::applyHttpAsyncClientBuilder)); - RestHighLevelClient writeClient = new RestHighLevelClient(RestClient.builder(properties.createHosts()) - .setRequestConfigCallback(properties::applyRequestConfigBuilder) - .setHttpClientConfigCallback(properties::applyHttpAsyncClientBuilder)); - return new ElasticRestClient(queryClient, writeClient); + RestHighLevelClient client = new RestHighLevelClient(RestClient.builder(properties.createHosts()) + .setRequestConfigCallback(properties::applyRequestConfigBuilder) + .setHttpClientConfigCallback(properties::applyHttpAsyncClientBuilder)); + return new ElasticRestClient(client, client); } } diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/enums/FieldDateFormat.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticDateFormat.java similarity index 80% rename from jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/enums/FieldDateFormat.java rename to jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticDateFormat.java index f8fa9717..dd221799 100644 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/enums/FieldDateFormat.java +++ b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticDateFormat.java @@ -4,6 +4,7 @@ import lombok.AllArgsConstructor; import lombok.Getter; import org.hswebframework.web.dict.EnumDict; +import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; @@ -14,7 +15,7 @@ import java.util.stream.Collectors; **/ @Getter @AllArgsConstructor -public enum FieldDateFormat implements EnumDict { +public enum ElasticDateFormat implements EnumDict { epoch_millis("epoch_millis", "毫秒"), epoch_second("epoch_second", "秒"), @@ -29,9 +30,13 @@ public enum FieldDateFormat implements EnumDict { private String text; - public static String getFormat(List dateFormats) { + public static String getFormat(ElasticDateFormat... dateFormats) { + return getFormat(Arrays.asList(dateFormats)); + } + + public static String getFormat(List dateFormats) { return getFormatStr(dateFormats.stream() - .map(FieldDateFormat::getValue) + .map(ElasticDateFormat::getValue) .collect(Collectors.toList()) ); } diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticPropertyType.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticPropertyType.java new file mode 100644 index 00000000..a1257729 --- /dev/null +++ b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticPropertyType.java @@ -0,0 +1,66 @@ +package org.jetlinks.community.elastic.search.enums; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.hswebframework.web.dict.EnumDict; +import org.hswebframework.web.exception.NotFoundException; +import org.jetlinks.core.metadata.DataType; +import org.jetlinks.core.metadata.types.*; +import org.springframework.util.StringUtils; + +import java.util.function.Supplier; + +@AllArgsConstructor +public enum ElasticPropertyType implements EnumDict { + + TEXT("text", "text", StringType::new), + BYTE("byte", "byte", () -> new IntType().min(Byte.MIN_VALUE).max(Byte.MAX_VALUE)), + SHORT("short", "short", () -> new IntType().min(Short.MIN_VALUE).max(Short.MAX_VALUE)), + INTEGER("int", "integer", IntType::new), + LONG("long", "long", LongType::new), + DATE("date", "date", DateTimeType::new), + HALF_FLOAT("half_float", "half_float", FloatType::new), + FLOAT("float", "float", FloatType::new), + DOUBLE("double", "double", DoubleType::new), + BOOLEAN("boolean", "boolean", BooleanType::new), + OBJECT("object", "object", ObjectType::new), + AUTO("auto", "auto", () -> null), + NESTED("nested", "nested", ObjectType::new), + IP("ip", "ip", LongType::new), + ATTACHMENT("attachment", "attachment", FileType::new), + KEYWORD("string", "keyword", StringType::new), + GEO_POINT("geo_point", "geo_point", GeoType::new); + + @Getter + private String text; + @Getter + private String value; + + private Supplier typeBuilder; + + public DataType getType() { + return typeBuilder.get(); + } + + public static ElasticPropertyType of(Object value) { + if (!StringUtils.isEmpty(value)) { + for (ElasticPropertyType elasticPropertyType : ElasticPropertyType.values()) { + if (elasticPropertyType.getValue().equals(value)) { + return elasticPropertyType; + } + } + } + return null; + } + + public static ElasticPropertyType ofJava(Object value) { + if (!StringUtils.isEmpty(value)) { + for (ElasticPropertyType elasticPropertyType : ElasticPropertyType.values()) { + if (elasticPropertyType.getText().equals(value)) { + return elasticPropertyType; + } + } + } + throw new NotFoundException("未找到数据类型为：" + value + "的枚举"); + } +} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/enums/FieldType.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/enums/FieldType.java deleted file mode 100644 index 62501fc6..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/enums/FieldType.java +++ /dev/null @@ -1,57 +0,0 @@ -package org.jetlinks.community.elastic.search.enums; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import org.hswebframework.web.dict.EnumDict; -import org.hswebframework.web.exception.NotFoundException; -import org.springframework.util.StringUtils; - -@AllArgsConstructor -@Getter -public enum FieldType implements EnumDict { - - TEXT("text", "text"), - BYTE("byte", "byte"), - SHORT("short", "short"), - INTEGER("int", "integer"), - LONG("long", "long"), - DATE("date", "date"), - HALF_FLOAT("half_float", "half_float"), - FLOAT("float", "float"), - DOUBLE("double", "double"), - BOOLEAN("boolean", "boolean"), - OBJECT("object", "object"), - AUTO("auto", "auto"), - NESTED("nested", "nested"), - IP("ip", "ip"), - ATTACHMENT("attachment", "attachment"), - KEYWORD("string", "keyword"); - - @Getter - private String text; - - @Getter - private String value; - - public static FieldType of(Object value) { - if (!StringUtils.isEmpty(value)) { - for (FieldType fieldType : FieldType.values()) { - if (fieldType.getValue().equals(value)) { - return fieldType; - } - } - } - throw new NotFoundException("未找到数据类型为：" + value + "的枚举"); - } - - public static FieldType ofJava(Object value) { - if (!StringUtils.isEmpty(value)) { - for (FieldType fieldType : FieldType.values()) { - if (fieldType.getText().equals(value)) { - return fieldType; - } - } - } - throw new NotFoundException("未找到数据类型为：" + value + "的枚举"); - } -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/enums/IndexPatternEnum.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/enums/IndexPatternEnum.java deleted file mode 100644 index 3e519969..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/enums/IndexPatternEnum.java +++ /dev/null @@ -1,18 +0,0 @@ -package org.jetlinks.community.elastic.search.enums; - -import lombok.AllArgsConstructor; -import lombok.Getter; - -/** - * @author bsetfeng - * @since 1.0 - **/ -@AllArgsConstructor -@Getter -public enum IndexPatternEnum { - - MONTH("month"), - DAY("day"); - - private String value; -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/enums/IndexStrategyEnum.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/enums/IndexStrategyEnum.java deleted file mode 100644 index 5d3d32eb..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/enums/IndexStrategyEnum.java +++ /dev/null @@ -1,17 +0,0 @@ -package org.jetlinks.community.elastic.search.enums; - -import lombok.AllArgsConstructor; -import lombok.Getter; - -/** - * @author bsetfeng - * @since 1.0 - **/ -@AllArgsConstructor -@Getter -public enum IndexStrategyEnum { - - SUFFIX("suffix"); - - private String value; -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/CreateIndex.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/CreateIndex.java deleted file mode 100644 index 3f49586d..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/CreateIndex.java +++ /dev/null @@ -1,68 +0,0 @@ -package org.jetlinks.community.elastic.search.index; - -import lombok.Getter; -import lombok.Setter; -import org.elasticsearch.client.indices.CreateIndexRequest; -import org.elasticsearch.common.settings.Settings; -import org.jetlinks.community.elastic.search.index.mapping.MappingFactory; -import org.jetlinks.community.elastic.search.index.setting.SettingFactory; - -import java.util.Collections; -import java.util.Map; - -/** - * @author bsetfeng - * @since 1.0 - **/ - -public class CreateIndex { - - @Getter - @Setter - private Map mapping; - - @Getter - @Setter - private Settings.Builder settings; - - private String index; - - @Deprecated - private String type; - - public CreateIndex addIndex(String index) { - this.index = index; - return this; - } - - @Deprecated - public CreateIndex addType(String type) { - this.type = type; - return this; - } - - public MappingFactory createMapping() { - return MappingFactory.createInstance(this); - } - - public SettingFactory createSettings() { - return SettingFactory.createInstance(this); - } - - - public CreateIndexRequest createIndexRequest() { - CreateIndexRequest request = new CreateIndexRequest(index); - request.mapping(Collections.singletonMap("properties", getMapping())); - if (settings != null) { - request.settings(settings); - } - return request; - } - - private CreateIndex() { - } - - public static CreateIndex createInstance() { - return new CreateIndex(); - } -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/DefaultElasticSearchIndexManager.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/DefaultElasticSearchIndexManager.java new file mode 100644 index 00000000..27b008b8 --- /dev/null +++ b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/DefaultElasticSearchIndexManager.java @@ -0,0 +1,62 @@ +package org.jetlinks.community.elastic.search.index; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; +import reactor.core.publisher.Mono; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +@Component +@ConfigurationProperties(prefix = "elasticsearch.index") +public class DefaultElasticSearchIndexManager implements ElasticSearchIndexManager { + + @Getter + @Setter + private String defaultStrategy = "direct"; + + @Getter + @Setter + private Map indexUseStrategy = new ConcurrentHashMap<>(); + + private Map strategies = new ConcurrentHashMap<>(); + + private Map indexMetadataStore = new ConcurrentHashMap<>(); + + public DefaultElasticSearchIndexManager(List strategies) { + strategies.forEach(this::registerStrategy); + } + + @Override + public Mono putIndex(ElasticSearchIndexMetadata index) { + return this.getIndexStrategy(index.getIndex()) + .flatMap(strategy -> strategy.putIndex(index)) + .doOnSuccess(metadata -> indexMetadataStore.put(index.getIndex(), index)); + } + + @Override + public Mono getIndexMetadata(String index) { + return Mono.justOrEmpty(indexMetadataStore.get(index)) + .switchIfEmpty(Mono.defer(() -> doLoadMetaData(index) + .doOnNext(metadata -> indexMetadataStore.put(metadata.getIndex(), metadata)))); + } + + protected Mono doLoadMetaData(String index) { + return getIndexStrategy(index) + .flatMap(strategy -> strategy.loadIndexMetadata(index)); + } + + @Override + public Mono getIndexStrategy(String index) { + return Mono.justOrEmpty(strategies.get(indexUseStrategy.getOrDefault(index.toLowerCase(), defaultStrategy))) + .switchIfEmpty(Mono.error(() -> new IllegalArgumentException("[" + index + "] 不支持任何索引策略"))); + } + + protected void registerStrategy(ElasticSearchIndexStrategy strategy) { + strategies.put(strategy.getId(), strategy); + } + +} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/DefaultElasticSearchIndexMetadata.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/DefaultElasticSearchIndexMetadata.java new file mode 100644 index 00000000..004d3dee --- /dev/null +++ b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/DefaultElasticSearchIndexMetadata.java @@ -0,0 +1,53 @@ +package org.jetlinks.community.elastic.search.index; + +import org.jetlinks.core.metadata.DataType; +import org.jetlinks.core.metadata.PropertyMetadata; +import org.jetlinks.core.metadata.SimplePropertyMetadata; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class DefaultElasticSearchIndexMetadata implements ElasticSearchIndexMetadata { + private String index; + + private Map properties = new HashMap<>(); + + public DefaultElasticSearchIndexMetadata(String index) { + this.index = index.toLowerCase().trim(); + } + + public DefaultElasticSearchIndexMetadata(String index, List properties) { + this(index); + properties.forEach(this::addProperty); + } + + @Override + public PropertyMetadata getProperty(String property) { + return properties.get(property); + } + + @Override + public String getIndex() { + return index; + } + + @Override + public List getProperties() { + return new ArrayList<>(properties.values()); + } + + public DefaultElasticSearchIndexMetadata addProperty(PropertyMetadata property) { + properties.put(property.getId(), property); + return this; + } + + public DefaultElasticSearchIndexMetadata addProperty(String property, DataType type) { + SimplePropertyMetadata metadata=new SimplePropertyMetadata(); + metadata.setValueType(type); + metadata.setId(property); + properties.put(property, metadata); + return this; + } +} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/DefaultIndexOperationService.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/DefaultIndexOperationService.java deleted file mode 100644 index d621c438..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/DefaultIndexOperationService.java +++ /dev/null @@ -1,136 +0,0 @@ -package org.jetlinks.community.elastic.search.index; - -import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.indices.*; -import org.hswebframework.web.bean.FastBeanCopier; -import org.jetlinks.community.elastic.search.ElasticRestClient; -import org.jetlinks.community.elastic.search.enums.FieldType; -import org.jetlinks.community.elastic.search.index.mapping.IndexMappingMetadata; -import org.jetlinks.community.elastic.search.index.mapping.IndicesMappingCenter; -import org.jetlinks.community.elastic.search.index.mapping.SingleMappingMetadata; -import org.jetlinks.community.elastic.search.service.IndexOperationService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.util.StringUtils; -import reactor.core.publisher.Mono; - -import java.util.HashMap; -import java.util.Map; - -/** - * @author bsetfeng - * @since 1.0 - **/ -@Service -@Slf4j -public class DefaultIndexOperationService implements IndexOperationService { - - private final ElasticRestClient restClient; - - private final IndicesMappingCenter indicesMappingCenter; - - @Autowired - public DefaultIndexOperationService(ElasticRestClient restClient, IndicesMappingCenter indicesMappingCenter) { - this.restClient = restClient; - this.indicesMappingCenter = indicesMappingCenter; - } - - - @Override - public Mono indexIsExists(String index) { - return Mono.create(sink -> { - restClient.getQueryClient().indices().existsAsync(new GetIndexRequest(index), RequestOptions.DEFAULT, new ActionListener() { - @Override - public void onResponse(Boolean aBoolean) { - sink.success(aBoolean); - } - - @Override - public void onFailure(Exception e) { - log.error("查询es index 是否存在失败", e); - sink.error(e); - } - }); - }); - } - - @Override - public Mono init(CreateIndexRequest request) { - return indexIsExists(request.index()) - .filter(bool -> !bool) - .flatMap(b -> Mono.create(sink -> { - restClient.getQueryClient().indices().createAsync(request, RequestOptions.DEFAULT, new ActionListener() { - @Override - public void onResponse(CreateIndexResponse createIndexResponse) { - sink.success(createIndexResponse.isAcknowledged()); - } - - @Override - public void onFailure(Exception e) { - sink.error(e); - } - }); - })); - - } - - - @Override - public Mono getIndexMappingMetadata(String index) { - return indicesMappingCenter.getIndexMappingMetaData(index) - .map(Mono::just).orElseGet(() -> - getIndexMapping(index) - .doOnNext(indicesMappingCenter::register)); - } - - private Mono getIndexMapping(String index) { - return indexIsExists(index) - .filter(Boolean::booleanValue) - .flatMap(bool -> Mono.create(sink -> { - if (bool) { - GetMappingsRequest mappingsRequest = new GetMappingsRequest(); - mappingsRequest.indices(index); - restClient.getQueryClient().indices().getMappingAsync(mappingsRequest, RequestOptions.DEFAULT, new ActionListener() { - @Override - public void onResponse(GetMappingsResponse getMappingsResponse) { - //index存在时 getMappingsResponse.mappings().get(index).getSourceAsMap().get("properties") 不会为空 - //sink.success(fieldMappingConvert(null, IndexMappingMetadata.getInstance(index), getMappingsResponse.mappings().get(index).getSourceAsMap().get("properties"))); - getMappingsResponse.mappings() - .forEach((k, v) -> sink.success(fieldMappingConvert(null, IndexMappingMetadata.getInstance(index), v.getSourceAsMap().get("properties")))); - } - - @Override - public void onFailure(Exception e) { - sink.error(e); - } - }); - } - })) - .switchIfEmpty(Mono.just(IndexMappingMetadata.getInstance(index))); - - } - - private IndexMappingMetadata fieldMappingConvert(String baseKey, IndexMappingMetadata indexMappingMetaData, Object properties) { - FastBeanCopier.copy(properties, new HashMap()) - .forEach((key, value) -> { - if (StringUtils.hasText(baseKey)) { - key = baseKey.concat(".").concat(key); - } - if (value instanceof Map) { - Map tempValue = FastBeanCopier.copy(value, new HashMap<>()); - Object childProperties = tempValue.get("properties"); - if (childProperties != null) { - fieldMappingConvert(key, indexMappingMetaData, childProperties); - return; - } - indexMappingMetaData.setMetadata(SingleMappingMetadata.builder() - .name(key) - .type(FieldType.of(tempValue.get("type"))) - .build()); - } - }); - return indexMappingMetaData; - } -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/ElasticIndex.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/ElasticIndex.java index e1dcf236..1638840b 100644 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/ElasticIndex.java +++ b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/ElasticIndex.java @@ -1,37 +1,9 @@ package org.jetlinks.community.elastic.search.index; -import java.util.function.Supplier; - /** * @version 1.0 **/ public interface ElasticIndex { - @Deprecated String getIndex(); - - @Deprecated - String getType(); - - default String getStandardIndex(){ - return getIndex().toLowerCase(); - } - - default String getStandardType(){ - return getType().toLowerCase(); - } - - static ElasticIndex createDefaultIndex(Supplier indexConsumer, Supplier typeConsumer) { - return new ElasticIndex() { - @Override - public String getIndex() { - return indexConsumer.get(); - } - - @Override - public String getType() { - return typeConsumer.get(); - } - }; - } } diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexManager.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexManager.java new file mode 100644 index 00000000..347f551e --- /dev/null +++ b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexManager.java @@ -0,0 +1,13 @@ +package org.jetlinks.community.elastic.search.index; + +import reactor.core.publisher.Mono; + +public interface ElasticSearchIndexManager { + + Mono putIndex(ElasticSearchIndexMetadata index); + + Mono getIndexMetadata(String index); + + Mono getIndexStrategy(String index); + +} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexMetadata.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexMetadata.java new file mode 100644 index 00000000..f9f424d2 --- /dev/null +++ b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexMetadata.java @@ -0,0 +1,28 @@ +package org.jetlinks.community.elastic.search.index; + +import org.jetlinks.core.metadata.PropertyMetadata; +import org.jetlinks.community.elastic.search.utils.ElasticSearchConverter; + +import java.util.List; +import java.util.Map; + +public interface ElasticSearchIndexMetadata { + + String getIndex(); + + List getProperties(); + + PropertyMetadata getProperty(String property); + + default Map convertToElastic(Map map) { + return ElasticSearchConverter.convertDataToElastic(map, getProperties()); + } + + default Map convertFromElastic(Map map) { + return ElasticSearchConverter.convertDataFromElastic(map, getProperties()); + } + + default ElasticSearchIndexMetadata newIndexName(String name) { + return new DefaultElasticSearchIndexMetadata(name, getProperties()); + } +} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexStrategy.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexStrategy.java new file mode 100644 index 00000000..365911be --- /dev/null +++ b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexStrategy.java @@ -0,0 +1,45 @@ +package org.jetlinks.community.elastic.search.index; + +import reactor.core.publisher.Mono; + +/** + * es 索引策略 + * + * @author zhouhao + * @version 1.0 + */ +public interface ElasticSearchIndexStrategy { + + /** + * 策略标识 + * + * @return ID + */ + String getId(); + + /** + * 获取用于获取保存数据的索引 + * + * @param index 原始索引名 + * @return 索引名 + */ + String getIndexForSave(String index); + + /** + * 获取用于搜索的索引 + * + * @param index 原始索引名 + * @return 索引名 + */ + String getIndexForSearch(String index); + + /** + * 更新索引 + * + * @param metadata 索引元数据 + * @return 更新结果 + */ + Mono putIndex(ElasticSearchIndexMetadata metadata); + + Mono loadIndexMetadata(String index); +} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/alias/DefaultIndexAliasOperationService.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/alias/DefaultIndexAliasOperationService.java deleted file mode 100644 index 7c87396c..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/alias/DefaultIndexAliasOperationService.java +++ /dev/null @@ -1,66 +0,0 @@ -package org.jetlinks.community.elastic.search.index.alias; - -import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; -import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; -import org.elasticsearch.action.support.master.AcknowledgedResponse; -import org.elasticsearch.client.RequestOptions; -import org.jetlinks.community.elastic.search.ElasticRestClient; -import org.jetlinks.community.elastic.search.service.IndexAliasOperationService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import reactor.core.publisher.Mono; - -/** - * @author bsetfeng - * @since 1.0 - **/ -@Service -@Slf4j -public class DefaultIndexAliasOperationService implements IndexAliasOperationService { - - private final ElasticRestClient restClient; - - @Autowired - public DefaultIndexAliasOperationService(ElasticRestClient restClient) { - this.restClient = restClient; - } - - - @Override - public Mono indexAliasIsExists(String alias) { - return Mono.create(sink -> { - GetAliasesRequest request = new GetAliasesRequest(alias); - restClient.getQueryClient().indices().existsAliasAsync(request, RequestOptions.DEFAULT, new ActionListener() { - @Override - public void onResponse(Boolean aBoolean) { - sink.success(aBoolean); - } - - @Override - public void onFailure(Exception e) { - sink.error(e); - } - }); - }); - } - - @Override - public Mono AddAlias(IndicesAliasesRequest request) { - return Mono.create(sink -> { - restClient.getQueryClient().indices().updateAliasesAsync(request, RequestOptions.DEFAULT, new ActionListener() { - @Override - public void onResponse(AcknowledgedResponse acknowledgedResponse) { - sink.success(acknowledgedResponse.isAcknowledged()); - } - - @Override - public void onFailure(Exception e) { - sink.error(e); - } - }); - }); - - } -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/mapping/IndexMappingMetadata.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/mapping/IndexMappingMetadata.java index 71746f59..1490a52d 100644 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/mapping/IndexMappingMetadata.java +++ b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/mapping/IndexMappingMetadata.java @@ -2,7 +2,7 @@ package org.jetlinks.community.elastic.search.index.mapping; import lombok.Getter; import lombok.Setter; -import org.jetlinks.community.elastic.search.enums.FieldType; +import org.jetlinks.community.elastic.search.enums.ElasticPropertyType; import java.util.HashMap; import java.util.List; @@ -33,14 +33,14 @@ public class IndexMappingMetadata { return metadata.get(fieldName); } - public List getMetaDataByType(FieldType type) { + public List getMetaDataByType(ElasticPropertyType type) { return getAllMetaData() .stream() .filter(singleMapping -> singleMapping.getType().equals(type)) .collect(Collectors.toList()); } - public Map getMetaDataByTypeToMap(FieldType type) { + public Map getMetaDataByTypeToMap(ElasticPropertyType type) { return getMetaDataByType(type) .stream() .collect(Collectors.toMap(SingleMappingMetadata::getName, Function.identity())); diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/mapping/IndicesMappingCenter.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/mapping/IndicesMappingCenter.java deleted file mode 100644 index 559adaba..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/mapping/IndicesMappingCenter.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.jetlinks.community.elastic.search.index.mapping; - -import org.springframework.stereotype.Component; - -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; - -/** - * @author bsetfeng - * @since 1.0 - **/ -@Component -public class IndicesMappingCenter { - - private Map indicesMapping = new ConcurrentHashMap<>(); - - - public Optional getIndexMappingMetaData(String index) { - return Optional.ofNullable(indicesMapping.get(index)); - } - - public void register(IndexMappingMetadata mappingMetaData) { - indicesMapping.put(mappingMetaData.getIndex(), mappingMetaData); - } -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/mapping/MappingFactory.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/mapping/MappingFactory.java deleted file mode 100644 index dcbe02eb..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/mapping/MappingFactory.java +++ /dev/null @@ -1,77 +0,0 @@ -package org.jetlinks.community.elastic.search.index.mapping; - -import org.hswebframework.web.exception.BusinessException; -import org.jetlinks.community.elastic.search.enums.FieldDateFormat; -import org.jetlinks.community.elastic.search.enums.FieldType; -import org.jetlinks.community.elastic.search.index.CreateIndex; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; - -/** - * @author bsetfeng - * @since 1.0 - **/ -public class MappingFactory { - - - private Map properties = new HashMap<>(); - - private Map filedMap = new HashMap<>(); - - private volatile boolean flag = true; - - private CreateIndex index; - - private MappingFactory(CreateIndex index) { - this.index = index; - } - - public MappingFactory addFieldName(String fieldName) { - continuityOperateHandle(!flag); - flag = false; - filedMap = new HashMap<>(); - properties.put(fieldName, filedMap); - return this; - } - - public MappingFactory addFieldType(FieldType type) { - continuityOperateHandle(flag); - filedMap.put("type", type.getValue()); - return this; - } - - public MappingFactory addFieldDateFormat(FieldDateFormat... dateFormats) { - continuityOperateHandle(flag); - filedMap.compute("format", (k, v) -> v == null ? FieldDateFormat.getFormat(Arrays.asList(dateFormats)) : v + "||" + FieldDateFormat.getFormat(Arrays.asList(dateFormats))); - return this; - } - - public MappingFactory addFieldDateFormat(String... dateFormats) { - continuityOperateHandle(flag); - filedMap.compute("format", (k, v) -> v == null ? FieldDateFormat.getFormatStr(Arrays.asList(dateFormats)) : v + "||" + FieldDateFormat.getFormatStr(Arrays.asList(dateFormats))); - return this; - } - - public MappingFactory commit() { - flag = true; - return this; - } - - public CreateIndex end() { - index.setMapping(properties); - return index; - } - - public static MappingFactory createInstance(CreateIndex index) { - return new MappingFactory(index); - } - - private void continuityOperateHandle(boolean inoperable) { - if (inoperable) { - throw new BusinessException("please exec commit() or addFiledName() later then operate"); - } - } - -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/mapping/SingleMappingMetadata.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/mapping/SingleMappingMetadata.java index a5a60d15..2349d26b 100644 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/mapping/SingleMappingMetadata.java +++ b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/mapping/SingleMappingMetadata.java @@ -1,8 +1,8 @@ package org.jetlinks.community.elastic.search.index.mapping; import lombok.*; -import org.jetlinks.community.elastic.search.enums.FieldDateFormat; -import org.jetlinks.community.elastic.search.enums.FieldType; +import org.jetlinks.community.elastic.search.enums.ElasticDateFormat; +import org.jetlinks.community.elastic.search.enums.ElasticPropertyType; /** * @author bsetfeng @@ -17,7 +17,7 @@ public class SingleMappingMetadata { private String name; - private FieldDateFormat format; + private ElasticDateFormat format; - private FieldType type; + private ElasticPropertyType type; } diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/setting/SettingFactory.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/setting/SettingFactory.java deleted file mode 100644 index 1d13886f..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/setting/SettingFactory.java +++ /dev/null @@ -1,40 +0,0 @@ -package org.jetlinks.community.elastic.search.index.setting; - -import org.elasticsearch.common.settings.Settings; -import org.jetlinks.community.elastic.search.index.CreateIndex; - -/** - * @author bsetfeng - * @since 1.0 - **/ -public class SettingFactory { - - private Settings.Builder settings = Settings.builder(); - - private CreateIndex index; - - private SettingFactory(CreateIndex index) { - this.index = index; - } - - public SettingFactory settingShards(Integer shards) { - settings.put("number_of_shards", shards); - return this; - } - - public SettingFactory settingReplicas(Integer replicas) { - settings.put("number_of_replicas", replicas); - return this; - } - - public CreateIndex end() { - index.setSettings(settings); - return index; - } - - public static SettingFactory createInstance(CreateIndex index) { - return new SettingFactory(index); - } - - -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/strategies/AbstractElasticSearchIndexStrategy.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/strategies/AbstractElasticSearchIndexStrategy.java new file mode 100644 index 00000000..3559fcf2 --- /dev/null +++ b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/strategies/AbstractElasticSearchIndexStrategy.java @@ -0,0 +1,187 @@ +package org.jetlinks.community.elastic.search.index.strategies; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.indices.*; +import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.jetlinks.core.metadata.DataType; +import org.jetlinks.core.metadata.PropertyMetadata; +import org.jetlinks.core.metadata.SimplePropertyMetadata; +import org.jetlinks.core.metadata.types.*; +import org.jetlinks.community.elastic.search.ElasticRestClient; +import org.jetlinks.community.elastic.search.enums.ElasticDateFormat; +import org.jetlinks.community.elastic.search.enums.ElasticPropertyType; +import org.jetlinks.community.elastic.search.index.DefaultElasticSearchIndexMetadata; +import org.jetlinks.community.elastic.search.index.ElasticSearchIndexMetadata; +import org.jetlinks.community.elastic.search.index.ElasticSearchIndexStrategy; +import org.jetlinks.community.elastic.search.utils.ReactorActionListener; +import reactor.core.publisher.Mono; + +import java.util.*; +import java.util.stream.Collectors; + +@AllArgsConstructor +@Slf4j +public abstract class AbstractElasticSearchIndexStrategy implements ElasticSearchIndexStrategy { + @Getter + private String id; + + protected ElasticRestClient client; + + private String wrapIndex(String index) { + return index.toLowerCase(); + } + + public Mono indexExists(String index) { + return ReactorActionListener.mono( + actionListener -> + client.getQueryClient() + .indices() + .existsAsync(new GetIndexRequest(wrapIndex(index)), RequestOptions.DEFAULT, actionListener)); + } + + public Mono doCreateIndex(ElasticSearchIndexMetadata metadata) { + return ReactorActionListener.mono( + actionListener -> client.getQueryClient() + .indices() + .createAsync(createIndexRequest(metadata), RequestOptions.DEFAULT, actionListener)) + .then(); + } + + protected Mono doPutIndex(ElasticSearchIndexMetadata metadata, + boolean justUpdateMapping) { + String index = wrapIndex(metadata.getIndex()); + return this.indexExists(index) + .flatMap(exists -> { + if (exists) { + return doLoadIndexMetadata(index) + .flatMap(oldMapping -> Mono.justOrEmpty(createPutMappingRequest(metadata, oldMapping))) + .flatMap(request -> ReactorActionListener.mono( + actionListener -> + client.getWriteClient() + .indices() + .putMappingAsync(request, RequestOptions.DEFAULT, actionListener))) + .then(); + } + if (justUpdateMapping) { + return Mono.empty(); + } + return doCreateIndex(metadata); + }); + } + + protected Mono doLoadIndexMetadata(String _index) { + String index = wrapIndex(_index); + return ReactorActionListener + .mono(listener -> client.getQueryClient() + .indices() + .getMappingAsync(new GetMappingsRequest().indices(index), RequestOptions.DEFAULT, listener)) + .flatMap(resp -> Mono.justOrEmpty(convertMetadata(index, resp.mappings().get(index)))); + } + + + public CreateIndexRequest createIndexRequest(ElasticSearchIndexMetadata metadata) { + CreateIndexRequest request = new CreateIndexRequest(wrapIndex(metadata.getIndex())); + request.mapping(Collections.singletonMap("properties", createElasticProperties(metadata.getProperties()))); + return request; + } + + private PutMappingRequest createPutMappingRequest(ElasticSearchIndexMetadata metadata, ElasticSearchIndexMetadata ignore) { + Map properties = createElasticProperties(metadata.getProperties()); + Map ignoreProperties = createElasticProperties(ignore.getProperties()); + for (Map.Entry en : ignoreProperties.entrySet()) { + log.debug("ignore update index [{}] mapping property:{},{}", wrapIndex(metadata.getIndex()), en.getKey(), en.getValue()); + properties.remove(en.getKey()); + } + if (properties.isEmpty()) { + log.debug("ignore update index [{}] mapping", wrapIndex(metadata.getIndex())); + return null; + } + PutMappingRequest request = new PutMappingRequest(wrapIndex(metadata.getIndex())); + request.source(Collections.singletonMap("properties", properties)); + return request; + } + + protected Map createElasticProperties(List metadata) { + if (metadata == null) { + return new HashMap<>(); + } + return metadata.stream() + .collect(Collectors.toMap(PropertyMetadata::getId, this::createElasticProperty)); + } + + protected Map createElasticProperty(PropertyMetadata metadata) { + Map property = new HashMap<>(); + + DataType type = metadata.getValueType(); + if (type instanceof DateTimeType) { + property.put("type", "date"); + property.put("format", ElasticDateFormat.getFormat(ElasticDateFormat.epoch_millis, ElasticDateFormat.simple_date, ElasticDateFormat.strict_date)); + } else if (type instanceof DoubleType) { + property.put("type", "double"); + } else if (type instanceof LongType) { + property.put("type", "long"); + } else if (type instanceof IntType) { + property.put("type", "integer"); + } else if (type instanceof FloatType) { + property.put("type", "float"); + } else if (type instanceof BooleanType) { + property.put("type", "boolean"); + } else if (type instanceof GeoType) { + property.put("type", "geo_point"); + } else if (type instanceof ObjectType) { + property.put("type", "nested"); + ObjectType objectType = ((ObjectType) type); + property.put("properties", createElasticProperties(objectType.getProperties())); + } else { + property.put("type", "keyword"); + } + return property; + } + + protected ElasticSearchIndexMetadata convertMetadata(String index, MappingMetaData metaData) { + Map response = metaData.getSourceAsMap(); + Object properties = response.get("properties"); + return new DefaultElasticSearchIndexMetadata(index, convertProperties(properties)); + } + + @SuppressWarnings("all") + protected List convertProperties(Object properties) { + if (properties == null) { + return new ArrayList<>(); + } + return ((Map>) properties) + .entrySet() + .stream() + .map(entry -> convertProperty(entry.getKey(), entry.getValue())) + .collect(Collectors.toList()); + } + + private PropertyMetadata convertProperty(String property, Map map) { + String type = String.valueOf(map.get("type")); + SimplePropertyMetadata metadata = new SimplePropertyMetadata(); + metadata.setId(property); + metadata.setName(property); + ElasticPropertyType elasticPropertyType = ElasticPropertyType.of(type); + if (null != elasticPropertyType) { + DataType dataType = elasticPropertyType.getType(); + if ((elasticPropertyType == ElasticPropertyType.OBJECT + || elasticPropertyType == ElasticPropertyType.NESTED) + && dataType instanceof ObjectType) { + @SuppressWarnings("all") + Map nestProperties = (Map) map.get("properties"); + if (null != nestProperties) { + ObjectType objectType = ((ObjectType) dataType); + objectType.setProperties(convertProperties(nestProperties)); + } + } + metadata.setValueType(dataType); + } else { + metadata.setValueType(new StringType()); + } + return metadata; + } +} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/strategies/DirectElasticSearchIndexStrategy.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/strategies/DirectElasticSearchIndexStrategy.java new file mode 100644 index 00000000..78ec13f9 --- /dev/null +++ b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/strategies/DirectElasticSearchIndexStrategy.java @@ -0,0 +1,35 @@ +package org.jetlinks.community.elastic.search.index.strategies; + +import org.jetlinks.community.elastic.search.ElasticRestClient; +import org.jetlinks.community.elastic.search.index.ElasticSearchIndexMetadata; +import org.springframework.stereotype.Component; +import reactor.core.publisher.Mono; + +@Component +public class DirectElasticSearchIndexStrategy extends AbstractElasticSearchIndexStrategy { + + public DirectElasticSearchIndexStrategy(ElasticRestClient client) { + super("direct", client); + } + + @Override + public String getIndexForSave(String index) { + return index; + } + + @Override + public String getIndexForSearch(String index) { + return index; + } + + @Override + public Mono putIndex(ElasticSearchIndexMetadata metadata) { + return doPutIndex(metadata, false); + } + + @Override + public Mono loadIndexMetadata(String index) { + return doLoadIndexMetadata(index); + } + +} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/strategies/TemplateElasticSearchIndexStrategy.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/strategies/TemplateElasticSearchIndexStrategy.java new file mode 100644 index 00000000..ae910783 --- /dev/null +++ b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/strategies/TemplateElasticSearchIndexStrategy.java @@ -0,0 +1,71 @@ +package org.jetlinks.community.elastic.search.index.strategies; + +import org.elasticsearch.action.admin.indices.alias.Alias; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.indices.GetIndexTemplatesRequest; +import org.elasticsearch.client.indices.GetIndexTemplatesResponse; +import org.elasticsearch.client.indices.PutIndexTemplateRequest; +import org.jetlinks.community.elastic.search.ElasticRestClient; +import org.jetlinks.community.elastic.search.index.ElasticSearchIndexMetadata; +import org.jetlinks.community.elastic.search.utils.ReactorActionListener; +import reactor.core.publisher.Mono; + +import java.util.Collections; +import java.util.List; + +public abstract class TemplateElasticSearchIndexStrategy extends AbstractElasticSearchIndexStrategy { + + public TemplateElasticSearchIndexStrategy(String id, ElasticRestClient client) { + super(id, client); + } + + protected String getTemplate(String index) { + return index.concat("_template"); + } + + protected String getAlias(String index) { + return index.concat("_alias"); + } + + protected List getIndexPatterns(String index) { + return Collections.singletonList(index.concat("*")); + } + + @Override + public abstract String getIndexForSave(String index); + + @Override + public String getIndexForSearch(String index) { + return getAlias(index); + } + + @Override + public Mono putIndex(ElasticSearchIndexMetadata metadata) { + return ReactorActionListener + .mono(listener -> client.getWriteClient() + .indices()//修改索引模版 + .putTemplateAsync(createIndexTemplateRequest(metadata), RequestOptions.DEFAULT, listener)) + //修改当前索引 + .then(doPutIndex(metadata.newIndexName(getIndexForSave(metadata.getIndex())), true)); + } + + protected PutIndexTemplateRequest createIndexTemplateRequest(ElasticSearchIndexMetadata metadata) { + String index = metadata.getIndex(); + PutIndexTemplateRequest request = new PutIndexTemplateRequest(getTemplate(index)); + request.alias(new Alias(getAlias(index))); + request.mapping(Collections.singletonMap("properties", createElasticProperties(metadata.getProperties()))); + request.patterns(getIndexPatterns(index)); + return request; + } + + @Override + public Mono loadIndexMetadata(String index) { + return ReactorActionListener + .mono(listener -> client.getQueryClient() + .indices() + .getIndexTemplateAsync(new GetIndexTemplatesRequest(getTemplate(index)), RequestOptions.DEFAULT, listener)) + .filter(resp -> resp.getIndexTemplates().size() > 0) + .flatMap(resp -> Mono.justOrEmpty(convertMetadata(index, resp.getIndexTemplates().get(0).mappings()))); + } +} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/strategies/TimeByMonthElasticSearchIndexStrategy.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/strategies/TimeByMonthElasticSearchIndexStrategy.java new file mode 100644 index 00000000..2f66f5ea --- /dev/null +++ b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/strategies/TimeByMonthElasticSearchIndexStrategy.java @@ -0,0 +1,28 @@ +package org.jetlinks.community.elastic.search.index.strategies; + +import org.hswebframework.utils.time.DateFormatter; +import org.jetlinks.community.elastic.search.ElasticRestClient; +import org.springframework.stereotype.Component; + +import java.util.Date; + +/** + * 按月对来划分索引策略 + * + * @author zhouhao + * @since 1.0 + */ +@Component +public class TimeByMonthElasticSearchIndexStrategy extends TemplateElasticSearchIndexStrategy { + + private final String format = "yyyy-MM"; + + public TimeByMonthElasticSearchIndexStrategy(ElasticRestClient client) { + super("time-by-month", client); + } + + @Override + public String getIndexForSave(String index) { + return index.concat("_").concat(DateFormatter.toString(new Date(), format)); + } +} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/template/DefaultIndexTemplateOperationService.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/template/DefaultIndexTemplateOperationService.java deleted file mode 100644 index fdcccbd7..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/template/DefaultIndexTemplateOperationService.java +++ /dev/null @@ -1,67 +0,0 @@ -package org.jetlinks.community.elastic.search.index.template; - -import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.support.master.AcknowledgedResponse; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.indices.IndexTemplatesExistRequest; -import org.elasticsearch.client.indices.PutIndexTemplateRequest; -import org.jetlinks.community.elastic.search.ElasticRestClient; -import org.jetlinks.community.elastic.search.service.IndexTemplateOperationService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import reactor.core.publisher.Mono; - -/** - * @author bsetfeng - * @since 1.0 - **/ -@Service -@Slf4j -public class DefaultIndexTemplateOperationService implements IndexTemplateOperationService { - - private final ElasticRestClient restClient; - - @Autowired - public DefaultIndexTemplateOperationService(ElasticRestClient restClient) { - this.restClient = restClient; - } - - - @Override - public Mono indexTemplateIsExists(String name) { - return Mono.create(sink -> { - IndexTemplatesExistRequest request = new IndexTemplatesExistRequest(name); - restClient.getQueryClient().indices().existsTemplateAsync(request, RequestOptions.DEFAULT, new ActionListener() { - @Override - public void onResponse(Boolean aBoolean) { - sink.success(aBoolean); - } - - @Override - public void onFailure(Exception e) { - sink.error(e); - } - }); - }); - } - - @Override - public Mono putTemplate(PutIndexTemplateRequest request) { - return indexTemplateIsExists(request.name()) - .filter(bool -> !bool) - .flatMap(b -> Mono.create(sink -> { - restClient.getQueryClient().indices().putTemplateAsync(request, RequestOptions.DEFAULT, new ActionListener() { - @Override - public void onResponse(AcknowledgedResponse acknowledgedResponse) { - sink.success(acknowledgedResponse.isAcknowledged()); - } - - @Override - public void onFailure(Exception e) { - sink.error(e); - } - }); - })); - } -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/DefaultIndexStrategyProvider.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/DefaultIndexStrategyProvider.java deleted file mode 100644 index dc06b978..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/DefaultIndexStrategyProvider.java +++ /dev/null @@ -1,50 +0,0 @@ -package org.jetlinks.community.elastic.search.manager; - -import lombok.Getter; -import org.jetlinks.community.elastic.search.enums.IndexPatternEnum; -import org.jetlinks.community.elastic.search.enums.IndexStrategyEnum; -import org.jetlinks.community.elastic.search.manager.entity.IndexStrategy; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; -import org.springframework.util.StringUtils; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * @author bsetfeng - * @since 1.0 - **/ -@Component -@Getter -public class DefaultIndexStrategyProvider implements IndexStrategyProvider { - - - @Value("${elasticsearch.time-series.index-strategy.suffix-month:}") - private List suffixMonthIndices; - - @Value("${elasticsearch.time-series.index-strategy.suffix-day:}") - private List suffixDayIndices; - - - @Value("${elasticsearch.time-series.index-strategy.format:yyyy-MM}") - private String format; - - @Value("${elasticsearch.time-series.index-strategy.connector:-}") - private String connector; - - @Override - public Map getIndexStrategies() { - Map indexStrategyMap = new HashMap<>(); - suffixMonthIndices - .stream() - .filter(StringUtils::hasText) - .forEach(s -> indexStrategyMap.put(s, new IndexStrategy(IndexStrategyEnum.SUFFIX.getValue(), IndexPatternEnum.MONTH.getValue()))); - suffixMonthIndices - .stream() - .filter(StringUtils::hasText) - .forEach(s -> indexStrategyMap.put(s, new IndexStrategy(IndexStrategyEnum.SUFFIX.getValue(), IndexPatternEnum.DAY.getValue()))); - return indexStrategyMap; - } -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/IndexManager.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/IndexManager.java deleted file mode 100644 index 91282e58..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/IndexManager.java +++ /dev/null @@ -1,20 +0,0 @@ -package org.jetlinks.community.elastic.search.manager; - -/** - * @author bsetfeng - * @since 1.0 - **/ -public interface IndexManager { - - IndexPatternManager getIndexPatternManager(); - - IndexStrategyManager getIndexStrategyManager(); - - String getFormat(); - - String getConnector(); - - default String getStandardIndex(String index) { - return getIndexStrategyManager().getStandardsIndex(index, getIndexPatternManager().getPattern(getFormat()), getConnector()); - } -} \ No newline at end of file diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/IndexPatternManager.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/IndexPatternManager.java deleted file mode 100644 index da1e5623..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/IndexPatternManager.java +++ /dev/null @@ -1,12 +0,0 @@ -package org.jetlinks.community.elastic.search.manager; - -/** - * @author bsetfeng - * @since 1.0 - **/ -public interface IndexPatternManager { - - String getName(); - - String getPattern(String format); -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/IndexStrategyManager.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/IndexStrategyManager.java deleted file mode 100644 index 3cdef6e3..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/IndexStrategyManager.java +++ /dev/null @@ -1,12 +0,0 @@ -package org.jetlinks.community.elastic.search.manager; - -/** - * @author bsetfeng - * @since 1.0 - **/ -public interface IndexStrategyManager { - - String getName(); - - String getStandardsIndex(String index, String pattern, String connector); -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/IndexStrategyProvider.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/IndexStrategyProvider.java deleted file mode 100644 index 3225f398..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/IndexStrategyProvider.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.jetlinks.community.elastic.search.manager; - -import org.jetlinks.community.elastic.search.manager.entity.IndexStrategy; - -import java.util.Map; - -/** - * @author bsetfeng - * @since 1.0 - **/ -public interface IndexStrategyProvider { - - /** - * index 时间格式。 ps: yyyy-MM-dd - * - * @return - */ - String getFormat(); - - /** - * index与策略串的连接符 - * - * @return - */ - default String connector() { - return "-"; - } - - /** - * @see StandardsIndexManagerCenter - * - * @return - */ - Map getIndexStrategies(); - - -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/StandardsIndexInit.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/StandardsIndexInit.java deleted file mode 100644 index 20fed923..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/StandardsIndexInit.java +++ /dev/null @@ -1,21 +0,0 @@ -package org.jetlinks.community.elastic.search.manager; - -import org.springframework.stereotype.Component; - -import java.util.concurrent.ScheduledExecutorService; - -/** - * @author bsetfeng - * @since 1.0 - **/ -@Component -public class StandardsIndexInit { - - private final ScheduledExecutorService executorService; - - public StandardsIndexInit(ScheduledExecutorService executorService) { - this.executorService = executorService; - } - - -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/StandardsIndexManager.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/StandardsIndexManager.java deleted file mode 100644 index 87487109..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/StandardsIndexManager.java +++ /dev/null @@ -1,21 +0,0 @@ -package org.jetlinks.community.elastic.search.manager; - -/** - * @author bsetfeng - * @since 1.0 - **/ -public interface StandardsIndexManager { - - String getStandardsIndex(String index); - - boolean indexIsChange(String index); - - boolean indexIsUpdate(String index); - - boolean standardsIndexIsUpdate(String standardsIndex); - - void addStandardsIndex(String standardsIndex); - - public void registerIndexManager(String index, IndexManager indexManager); - -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/StandardsIndexManagerCenter.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/StandardsIndexManagerCenter.java deleted file mode 100644 index 2e172272..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/StandardsIndexManagerCenter.java +++ /dev/null @@ -1,124 +0,0 @@ -package org.jetlinks.community.elastic.search.manager; - -import org.springframework.beans.BeansException; -import org.springframework.beans.factory.config.BeanPostProcessor; -import org.springframework.boot.CommandLineRunner; -import org.springframework.stereotype.Component; - -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentSkipListSet; - -/** - * @author bsetfeng - * @since 1.0 - **/ -@Component -public class StandardsIndexManagerCenter implements StandardsIndexManager, BeanPostProcessor, CommandLineRunner { - - - Map managerMap = new ConcurrentHashMap<>(); - - Map patternManagerMap = new ConcurrentHashMap<>(); - - Map strategyManagerMap = new ConcurrentHashMap<>(); - - Set standardsIndices = new ConcurrentSkipListSet<>(); - - private final List indexStrategyProviders; - - public StandardsIndexManagerCenter(List indexStrategyProviders) { - this.indexStrategyProviders = indexStrategyProviders; - } - - public void registerIndexPatternManager(IndexPatternManager patternManager) { - patternManagerMap.put(patternManager.getName(), patternManager); - } - - public void registerIndexStrategyManager(IndexStrategyManager strategyManager) { - strategyManagerMap.put(strategyManager.getName(), strategyManager); - } - - @Override - public void registerIndexManager(String index, IndexManager indexManager) { - managerMap.put(index, indexManager); - } - - - @Override - public String getStandardsIndex(String index) { - return Optional.ofNullable(managerMap.get(index)) - .map(m -> { - String standardsIndex = m.getStandardIndex(index); - standardsIndices.add(standardsIndex); - return standardsIndex; - }) - .orElse(index); - } - - @Override - public boolean indexIsChange(String index) { - return managerMap.containsKey(index); - } - - @Override - public boolean indexIsUpdate(String index) { - return Optional.ofNullable(managerMap.get(index)) - .map(m -> !standardsIndices.contains(m.getStandardIndex(index))) - .orElse(false); - } - - - @Override - public boolean standardsIndexIsUpdate(String standardsIndex) { - return !standardsIndices.contains(standardsIndex); - } - - @Override - public void addStandardsIndex(String standardsIndex) { - standardsIndices.add(standardsIndex); - } - - @Override - public Object postProcessAfterInitialization(Object bean, String beanName) throws BeansException { - if (bean instanceof IndexPatternManager) { - registerIndexPatternManager((IndexPatternManager) bean); - } - if (bean instanceof IndexStrategyManager) { - registerIndexStrategyManager((IndexStrategyManager) bean); - } - return bean; - } - - - @Override - public void run(String... args) throws Exception { - indexStrategyProviders.forEach(provider -> { - provider.getIndexStrategies() - .forEach((k, v) -> managerMap.put(k, new IndexManager() { - @Override - public IndexPatternManager getIndexPatternManager() { - return patternManagerMap.get(v.getPatternName()); - } - - @Override - public IndexStrategyManager getIndexStrategyManager() { - return strategyManagerMap.get(v.getStrategyName()); - } - - @Override - public String getFormat() { - return provider.getFormat(); - } - - @Override - public String getConnector() { - return provider.connector(); - } - })); - }); - } -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/entity/IndexStrategy.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/entity/IndexStrategy.java deleted file mode 100644 index 46819ac1..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/entity/IndexStrategy.java +++ /dev/null @@ -1,19 +0,0 @@ -package org.jetlinks.community.elastic.search.manager.entity; - -import lombok.*; - -/** - * @author bsetfeng - * @since 1.0 - **/ -@Getter -@Setter -@AllArgsConstructor -@NoArgsConstructor -@Builder -public class IndexStrategy { - - private String strategyName; - - private String patternName; -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/strategy/IndexDayPattern.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/strategy/IndexDayPattern.java deleted file mode 100644 index 8bcd8fd1..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/strategy/IndexDayPattern.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.jetlinks.community.elastic.search.manager.strategy; - -import org.jetlinks.community.elastic.search.enums.IndexPatternEnum; -import org.jetlinks.community.elastic.search.manager.IndexPatternManager; -import org.springframework.stereotype.Component; - -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; - -/** - * @author bsetfeng - * @since 1.0 - **/ -@Component -public class IndexDayPattern implements IndexPatternManager { - @Override - public String getName() { - return IndexPatternEnum.DAY.getValue(); - } - - @Override - public String getPattern(String format) { - LocalDateTime localDateTime = LocalDateTime.now(); - return localDateTime.format(DateTimeFormatter.ofPattern(format)); - } -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/strategy/IndexMonthPattern.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/strategy/IndexMonthPattern.java deleted file mode 100644 index f0e933d2..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/strategy/IndexMonthPattern.java +++ /dev/null @@ -1,27 +0,0 @@ -package org.jetlinks.community.elastic.search.manager.strategy; - -import org.jetlinks.community.elastic.search.enums.IndexPatternEnum; -import org.jetlinks.community.elastic.search.manager.IndexPatternManager; -import org.springframework.stereotype.Component; - -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; - -/** - * @author bsetfeng - * @since 1.0 - **/ -@Component -public class IndexMonthPattern implements IndexPatternManager { - - @Override - public String getName() { - return IndexPatternEnum.MONTH.getValue(); - } - - @Override - public String getPattern(String format) { - LocalDateTime localDateTime = LocalDateTime.now(); - return localDateTime.format(DateTimeFormatter.ofPattern(format)); - } -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/strategy/IndexSuffixStrategy.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/strategy/IndexSuffixStrategy.java deleted file mode 100644 index e5e269ea..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/manager/strategy/IndexSuffixStrategy.java +++ /dev/null @@ -1,24 +0,0 @@ -package org.jetlinks.community.elastic.search.manager.strategy; - -import org.jetlinks.community.elastic.search.enums.IndexStrategyEnum; -import org.jetlinks.community.elastic.search.manager.IndexStrategyManager; -import org.springframework.stereotype.Component; - -/** - * @author bsetfeng - * @since 1.0 - **/ -@Component -public class IndexSuffixStrategy implements IndexStrategyManager { - - - @Override - public String getName() { - return IndexStrategyEnum.SUFFIX.getValue(); - } - - @Override - public String getStandardsIndex(String index, String pattern, String connector) { - return index.concat(connector).concat(pattern); - } -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/parser/AbstractQueryParamTranslateService.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/parser/AbstractQueryParamTranslateService.java deleted file mode 100644 index 02ad6e17..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/parser/AbstractQueryParamTranslateService.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.jetlinks.community.elastic.search.parser; - -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.sort.SortOrder; -import org.hswebframework.ezorm.core.param.QueryParam; -import org.jetlinks.community.elastic.search.index.mapping.IndexMappingMetadata; -import org.springframework.util.StringUtils; - -/** - * @author bsetfeng - * @since 1.0 - **/ -public abstract class AbstractQueryParamTranslateService implements QueryParamTranslateService { - - @Override - public SearchSourceBuilder translate(QueryParam queryParam, IndexMappingMetadata mappingMetaData) { - SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); - if (queryParam.isPaging()) { - sourceBuilder.from(queryParam.getPageIndex() * queryParam.getPageSize()); - sourceBuilder.size(queryParam.getPageSize()); - } - queryParam.getSorts() - .forEach(sort -> { - if (!StringUtils.isEmpty(sort.getName())) { - sourceBuilder.sort(sort.getName(), SortOrder.fromString(sort.getOrder())); - } - - }); - return sourceBuilder.query(queryBuilder(queryParam, mappingMetaData)); - } - - protected abstract QueryBuilder queryBuilder(QueryParam queryParam, IndexMappingMetadata mappingMetaData); - -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/parser/DefaultLinkTypeParser.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/parser/DefaultLinkTypeParser.java index a43d05ed..87a5f95f 100644 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/parser/DefaultLinkTypeParser.java +++ b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/parser/DefaultLinkTypeParser.java @@ -5,7 +5,6 @@ import org.elasticsearch.index.query.QueryBuilders; import org.hswebframework.ezorm.core.param.Term; import org.springframework.stereotype.Component; -import java.util.LinkedList; import java.util.List; import java.util.function.Consumer; @@ -20,13 +19,10 @@ public class DefaultLinkTypeParser implements LinkTypeParser { @Override public BoolQueryBuilder process(Term term, Consumer consumer, BoolQueryBuilder queryBuilders) { - - if ("or".equalsIgnoreCase(term.getType().name())) { + if (term.getType() == Term.Type.or) { handleOr(queryBuilders, term, consumer); - } else if ("and".equalsIgnoreCase(term.getType().name())) { - handleAnd(queryBuilders, term, consumer); } else { - throw new UnsupportedOperationException("不支持的查询连接类型,term.getType:" + term.getType().name()); + handleAnd(queryBuilders, term, consumer); } return queryBuilders; } @@ -37,7 +33,7 @@ public class DefaultLinkTypeParser implements LinkTypeParser { parser.process(() -> term, queryBuilders::should); } else { BoolQueryBuilder nextQuery = QueryBuilders.boolQuery(); - List terms = ( term.getTerms()); + List terms = (term.getTerms()); terms.forEach(t -> process(t, consumer, nextQuery)); queryBuilders.should(nextQuery); } @@ -45,7 +41,7 @@ public class DefaultLinkTypeParser implements LinkTypeParser { private void handleAnd(BoolQueryBuilder queryBuilders, Term term, Consumer consumer) { consumer.accept(term); - if (term.getTerms().isEmpty()&& term.getValue() != null) { + if (term.getTerms().isEmpty() && term.getValue() != null) { parser.process(() -> term, queryBuilders::must); } else { BoolQueryBuilder nextQuery = QueryBuilders.boolQuery(); @@ -54,4 +50,6 @@ public class DefaultLinkTypeParser implements LinkTypeParser { queryBuilders.must(nextQuery); } } + + } diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/parser/DefaultQueryParamTranslateService.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/parser/DefaultQueryParamTranslateService.java deleted file mode 100644 index 02362bf8..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/parser/DefaultQueryParamTranslateService.java +++ /dev/null @@ -1,52 +0,0 @@ -package org.jetlinks.community.elastic.search.parser; - -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.hswebframework.ezorm.core.param.QueryParam; -import org.hswebframework.ezorm.core.param.Term; -import org.jetlinks.community.elastic.search.enums.FieldType; -import org.jetlinks.community.elastic.search.index.mapping.IndexMappingMetadata; -import org.jetlinks.community.elastic.search.index.mapping.SingleMappingMetadata; -import org.jetlinks.community.elastic.search.utils.DateTimeUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - -import java.util.List; - -/** - * @author bsetfeng - * @since 1.0 - **/ -@Component -public class DefaultQueryParamTranslateService extends AbstractQueryParamTranslateService { - - - private final LinkTypeParser parser; - - @Value("${jetlinks.system.formats:yyyy-MM-dd HH:mm:ss}") - private List formats; - - @Autowired - public DefaultQueryParamTranslateService(LinkTypeParser parser) { - this.parser = parser; - } - - @Override - protected QueryBuilder queryBuilder(QueryParam queryParam, IndexMappingMetadata mappingMetaData) { - BoolQueryBuilder queryBuilders = QueryBuilders.boolQuery(); - queryParam.getTerms() - .forEach(term -> parser.process(term, t -> - dateTypeHandle(t, mappingMetaData.getMetaData(t.getColumn())), queryBuilders)); - return queryBuilders; - } - - - private void dateTypeHandle(Term term, SingleMappingMetadata singleMappingMetaData) { - if (singleMappingMetaData == null) return; - if (singleMappingMetaData.getType().equals(FieldType.DATE)) { - term.setValue(DateTimeUtils.formatDateToTimestamp(term.getValue(), formats)); - } - } -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/parser/QueryParamTranslateService.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/parser/QueryParamTranslateService.java deleted file mode 100644 index 0ff4937b..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/parser/QueryParamTranslateService.java +++ /dev/null @@ -1,14 +0,0 @@ -package org.jetlinks.community.elastic.search.parser; - -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.hswebframework.ezorm.core.param.QueryParam; -import org.jetlinks.community.elastic.search.index.mapping.IndexMappingMetadata; - -/** - * @author bsetfeng - * @since 1.0 - **/ -public interface QueryParamTranslateService { - - SearchSourceBuilder translate(QueryParam queryParam, IndexMappingMetadata metaData); -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/AggregationService.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/AggregationService.java index ff89c7ac..8477ebdb 100644 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/AggregationService.java +++ b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/AggregationService.java @@ -6,15 +6,25 @@ import org.jetlinks.community.elastic.search.aggreation.bucket.BucketResponse; import org.jetlinks.community.elastic.search.aggreation.metrics.MetricsAggregationStructure; import org.jetlinks.community.elastic.search.aggreation.metrics.MetricsResponse; import org.jetlinks.community.elastic.search.index.ElasticIndex; +import org.jetlinks.community.timeseries.query.AggregationQueryParam; +import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; +import java.util.Map; + /** * @author bsetfeng * @since 1.0 **/ public interface AggregationService { - Mono metricsAggregation(QueryParam queryParam, MetricsAggregationStructure structure, ElasticIndex provider); + Mono metricsAggregation(String index, QueryParam queryParam, MetricsAggregationStructure structure); - Mono bucketAggregation(QueryParam queryParam, BucketAggregationsStructure structure, ElasticIndex provider); + Mono bucketAggregation(String index, QueryParam queryParam, BucketAggregationsStructure structure); + + Flux> aggregation(String index, AggregationQueryParam queryParam); + + default Flux> aggregation(ElasticIndex index, AggregationQueryParam queryParam) { + return aggregation(index.getIndex(), queryParam); + } } diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/DefaultElasticSearchService.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/DefaultElasticSearchService.java index a31f0ea2..abbb8ab4 100644 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/DefaultElasticSearchService.java +++ b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/DefaultElasticSearchService.java @@ -1,13 +1,9 @@ package org.jetlinks.community.elastic.search.service; import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONObject; import lombok.AllArgsConstructor; import lombok.Getter; -import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; @@ -19,26 +15,31 @@ import org.elasticsearch.client.core.CountRequest; import org.elasticsearch.client.core.CountResponse; import org.elasticsearch.common.xcontent.XContentType; import org.hswebframework.ezorm.core.param.QueryParam; +import org.hswebframework.utils.time.DateFormatter; +import org.hswebframework.utils.time.DefaultDateFormatter; import org.hswebframework.web.api.crud.entity.PagerResult; import org.jetlinks.core.utils.FluxUtils; import org.jetlinks.community.elastic.search.ElasticRestClient; -import org.jetlinks.community.elastic.search.index.ElasticIndex; -import org.jetlinks.community.elastic.search.index.mapping.IndexMappingMetadata; -import org.jetlinks.community.elastic.search.parser.QueryParamTranslateService; +import org.jetlinks.community.elastic.search.index.ElasticSearchIndexManager; +import org.jetlinks.community.elastic.search.utils.ElasticSearchConverter; +import org.jetlinks.community.elastic.search.utils.ReactorActionListener; import org.reactivestreams.Publisher; import org.springframework.stereotype.Service; -import org.springframework.util.StringUtils; -import reactor.core.publisher.*; -import reactor.core.scheduler.Schedulers; +import reactor.core.publisher.BufferOverflowStrategy; +import reactor.core.publisher.Flux; +import reactor.core.publisher.FluxSink; +import reactor.core.publisher.Mono; +import reactor.util.function.Tuples; import javax.annotation.PreDestroy; import java.time.Duration; import java.util.*; import java.util.function.Function; +import java.util.regex.Pattern; import java.util.stream.Collectors; /** - * @author bsetfeng + * @author zhouhao * @since 1.0 **/ @Service @@ -47,26 +48,35 @@ public class DefaultElasticSearchService implements ElasticSearchService { private final ElasticRestClient restClient; - private final IndexOperationService indexOperationService; - - private final QueryParamTranslateService translateService; + private final ElasticSearchIndexManager indexManager; FluxSink sink; + static { + DateFormatter.supportFormatter.add(new DefaultDateFormatter(Pattern.compile("[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.+"), "yyyy-MM-dd'T'HH:mm:ss.SSSZ")); + } + public DefaultElasticSearchService(ElasticRestClient restClient, - QueryParamTranslateService translateService, - IndexOperationService indexOperationService) { + ElasticSearchIndexManager indexManager) { this.restClient = restClient; - this.translateService = translateService; - this.indexOperationService = indexOperationService; init(); + this.indexManager = indexManager; } + public Flux query(String index, QueryParam queryParam, Function, T> mapper) { + return doSearch(createSearchRequest(queryParam, index)) + .flatMapIterable(response -> translate(mapper, response)) + .onErrorResume(err -> { + log.error("query elastic error", err); + return Mono.empty(); + }); + } + @Override - public Mono> queryPager(ElasticIndex index, QueryParam queryParam, Class type) { - return query(searchRequestStructure(queryParam, index)) - .map(response -> translatePageResult(type, queryParam, response)) + public Mono> queryPager(String index, QueryParam queryParam, Function, T> mapper) { + return doSearch(createSearchRequest(queryParam, index)) + .map(response -> translatePageResult(mapper, queryParam, response)) .switchIfEmpty(Mono.just(PagerResult.empty())) .onErrorReturn(err -> { log.error("query elastic error", err); @@ -75,18 +85,10 @@ public class DefaultElasticSearchService implements ElasticSearchService { } @Override - public Flux query(ElasticIndex index, QueryParam queryParam, Class type) { - return query(searchRequestStructure(queryParam, index)) - .flatMapIterable(response -> translate(type, response)) - .onErrorResume(err -> { - log.error("query elastic error", err); - return Flux.empty(); - }); - } - - @Override - public Mono count(ElasticIndex index, QueryParam queryParam) { - return countQuery(countRequestStructure(queryParam, index)) + public Mono count(String index, QueryParam queryParam) { + QueryParam param=queryParam.clone(); + param.setPaging(false); + return doCount(createCountRequest(param, index)) .map(CountResponse::getCount) .defaultIfEmpty(0L) .onErrorReturn(err -> { @@ -95,16 +97,15 @@ public class DefaultElasticSearchService implements ElasticSearchService { }, 0L); } - @Override - public Mono commit(ElasticIndex index, T payload) { + public Mono commit(String index, T payload) { return Mono.fromRunnable(() -> { sink.next(new Buffer(index, payload)); }); } @Override - public Mono commit(ElasticIndex index, Collection payload) { + public Mono commit(String index, Collection payload) { return Mono.fromRunnable(() -> { for (T t : payload) { sink.next(new Buffer(index, t)); @@ -113,7 +114,7 @@ public class DefaultElasticSearchService implements ElasticSearchService { } @Override - public Mono commit(ElasticIndex index, Publisher data) { + public Mono commit(String index, Publisher data) { return Flux.from(data) .flatMap(d -> commit(index, d)) .then(); @@ -148,34 +149,51 @@ public class DefaultElasticSearchService implements ElasticSearchService { @AllArgsConstructor @Getter static class Buffer { - ElasticIndex index; + String index; Object payload; } + private Mono getIndexForSave(String index) { + return indexManager + .getIndexStrategy(index) + .map(strategy -> strategy.getIndexForSave(index)); + + } + + private Mono getIndexForSearch(String index) { + return indexManager + .getIndexStrategy(index) + .map(strategy -> strategy.getIndexForSearch(index)); + + } + protected Mono doSave(Collection buffers) { return Flux.fromIterable(buffers) .collect(Collectors.groupingBy(Buffer::getIndex)) - .map(Map::entrySet) - .flatMapIterable(Function.identity()) + .flatMapMany(map -> Flux + .fromIterable(map.entrySet()) + .flatMap(e -> + this.getIndexForSave(e.getKey()) + .zipWith(indexManager.getIndexMetadata(e.getKey()), (index, metadata) -> Tuples.of(index, metadata, e.getValue())))) .map(entry -> { - ElasticIndex index = entry.getKey(); - BulkRequest bulkRequest = new BulkRequest(index.getStandardIndex(), index.getStandardType()); - for (Buffer buffer : entry.getValue()) { + String index = entry.getT1(); + BulkRequest bulkRequest = new BulkRequest(index, "_doc"); + for (Buffer buffer : entry.getT3()) { IndexRequest request = new IndexRequest(); Object o = JSON.toJSON(buffer.getPayload()); if (o instanceof Map) { - request.source((Map) o); + request.source(ElasticSearchConverter.convertDataToElastic((Map) o, entry.getT2().getProperties())); } else { request.source(o.toString(), XContentType.JSON); } bulkRequest.add(request); } - entry.getValue().clear(); + entry.getT3().clear(); return bulkRequest; }) .flatMap(bulkRequest -> - Mono.create(sink -> + Mono.create(sink -> restClient.getWriteClient() .bulkAsync(bulkRequest, RequestOptions.DEFAULT, new ActionListener() { @Override @@ -184,7 +202,7 @@ public class DefaultElasticSearchService implements ElasticSearchService { sink.error(new RuntimeException("保存ElasticSearch数据失败:" + responses.buildFailureMessage())); return; } - sink.success(buffers.size()); + sink.success(1); } @Override @@ -192,97 +210,68 @@ public class DefaultElasticSearchService implements ElasticSearchService { sink.error(e); } }))) - .collect(Collectors.summingInt(Integer::intValue)); + .then(Mono.just(buffers.size())); } - private PagerResult translatePageResult(Class clazz, QueryParam param, SearchResponse response) { + private PagerResult translatePageResult(Function, T> mapper, QueryParam param, SearchResponse response) { long total = response.getHits().getTotalHits(); - return PagerResult.of((int) total, translate(clazz, response), param); + return PagerResult.of((int) total, translate(mapper, response), param); } - private List translate(Class clazz, SearchResponse response) { - // TODO: 2020/1/18 临时代码 + private List translate(Function, T> mapper, SearchResponse response) { return Arrays.stream(response.getHits().getHits()) .map(hit -> { Map hitMap = hit.getSourceAsMap(); hitMap.put("id", hit.getId()); - return JSON.toJavaObject(new JSONObject(hitMap), clazz); + return mapper.apply(hitMap); }) .collect(Collectors.toList()); } - private Mono query(Mono requestMono) { - return requestMono.flatMap((request) -> - Mono.create(sink -> restClient - .getQueryClient() - .searchAsync(request, RequestOptions.DEFAULT, translatorActionListener(sink)))) + private Mono doSearch(Mono requestMono) { + return requestMono.flatMap((request) -> + ReactorActionListener + .mono(listener -> + restClient + .getQueryClient() + .searchAsync(request, RequestOptions.DEFAULT, listener))) .onErrorResume(err -> { log.error("query elastic error", err); return Mono.empty(); }); } - private Mono countQuery(Mono requestMono) { - return requestMono.flatMap((request) -> - Mono.create(sink -> restClient - .getQueryClient() - .countAsync(request, RequestOptions.DEFAULT, translatorActionListener(sink)))) + private Mono doCount(Mono requestMono) { + return requestMono.flatMap((request) -> + ReactorActionListener + .mono(listener -> + restClient + .getQueryClient() + .countAsync(request, RequestOptions.DEFAULT, listener))) .onErrorResume(err -> { log.error("query elastic error", err); return Mono.empty(); }); } - private ActionListener translatorActionListener(MonoSink sink) { - return new ActionListener() { - @Override - public void onResponse(T response) { - sink.success(response); - } - - @Override - public void onFailure(Exception e) { - if (e instanceof ElasticsearchException) { - if (((ElasticsearchException) e).status().getStatus() == 404) { - sink.success(); - return; - } else if (((ElasticsearchException) e).status().getStatus() == 400) { - sink.error(new ElasticsearchParseException("查询参数格式错误", e)); - } - } - sink.error(e); - } - }; + private Mono createSearchRequest(QueryParam queryParam, String index) { + return indexManager + .getIndexMetadata(index) + .map(metadata -> ElasticSearchConverter.convertSearchSourceBuilder(queryParam, metadata)) + .switchIfEmpty(Mono.fromSupplier(() -> ElasticSearchConverter.convertSearchSourceBuilder(queryParam, null))) + .flatMap(builder -> this.getIndexForSearch(index) + .map(idx -> new SearchRequest(idx).source(builder).types("_doc"))); } - private Mono searchRequestStructure(QueryParam queryParam, ElasticIndex provider) { - return indexOperationService.getIndexMappingMetadata(provider.getStandardIndex()) - .switchIfEmpty(Mono.just(IndexMappingMetadata.getInstance(provider.getStandardIndex()))) - .map(metadata -> { - SearchRequest request = new SearchRequest(provider.getStandardIndex()) - .source(translateService.translate(queryParam, metadata)); - if (StringUtils.hasText(provider.getStandardType())) { - request.types(provider.getStandardType()); - } - return request; - }) - .doOnNext(searchRequest -> log.debug("查询index：{},es查询参数:{}", provider.getStandardIndex(), searchRequest.source().toString())) - .onErrorResume(err -> { - log.error("query index error", err); - return Mono.empty(); - }); - } - - private Mono countRequestStructure(QueryParam queryParam, ElasticIndex provider) { + private Mono createCountRequest(QueryParam queryParam, String index) { QueryParam tempQueryParam = queryParam.clone(); tempQueryParam.setPaging(false); tempQueryParam.setSorts(Collections.emptyList()); - return indexOperationService.getIndexMappingMetadata(provider.getStandardIndex()) - .map(metadata -> new CountRequest(provider.getStandardIndex()) - .source(translateService.translate(tempQueryParam, metadata))) - .onErrorResume(err -> { - log.error("query index error", err); - return Mono.empty(); - }); + return indexManager + .getIndexMetadata(index) + .map(metadata -> ElasticSearchConverter.convertSearchSourceBuilder(queryParam, metadata)) + .switchIfEmpty(Mono.fromSupplier(() -> ElasticSearchConverter.convertSearchSourceBuilder(queryParam, null))) + .flatMap(builder -> this.getIndexForSearch(index) + .map(idx -> new CountRequest(idx).source(builder))); } } diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/ElasticSearchService.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/ElasticSearchService.java index ef38ba9e..c47e76c1 100644 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/ElasticSearchService.java +++ b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/ElasticSearchService.java @@ -2,25 +2,68 @@ package org.jetlinks.community.elastic.search.service; import org.hswebframework.ezorm.core.param.QueryParam; import org.hswebframework.web.api.crud.entity.PagerResult; +import org.hswebframework.web.bean.FastBeanCopier; import org.jetlinks.community.elastic.search.index.ElasticIndex; import org.reactivestreams.Publisher; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import java.util.Collection; +import java.util.Map; +import java.util.function.Function; public interface ElasticSearchService { + Mono> queryPager(String index, QueryParam queryParam, Function, T> mapper); - Mono> queryPager(ElasticIndex index, QueryParam queryParam, Class type); + Flux query(String index, QueryParam queryParam, Function, T> mapper); - Flux query(ElasticIndex index, QueryParam queryParam, Class type); + Mono count(String index, QueryParam queryParam); - Mono count(ElasticIndex index, QueryParam queryParam); + Mono commit(String index, T payload); - Mono commit(ElasticIndex index, T payload); + Mono commit(String index, Collection payload); - Mono commit(ElasticIndex index, Collection payload); + Mono commit(String index, Publisher data); + + default Flux query(String index, QueryParam queryParam, Class type) { + return query(index, queryParam, map -> FastBeanCopier.copy(map, type)); + } + + default Mono> queryPager(String index, QueryParam queryParam, Class type) { + return queryPager(index, queryParam, map -> FastBeanCopier.copy(map, type)); + } + + default Mono> queryPager(ElasticIndex index, QueryParam queryParam, Class type) { + return queryPager(index.getIndex(), queryParam, map -> FastBeanCopier.copy(map, type)); + } + + default Mono> queryPager(ElasticIndex index, QueryParam queryParam, Function, T> mapper) { + return queryPager(index.getIndex(), queryParam, mapper); + } + + default Flux query(ElasticIndex index, QueryParam queryParam, Class type) { + return query(index.getIndex(), queryParam, map -> FastBeanCopier.copy(map, type)); + } + + default Flux query(ElasticIndex index, QueryParam queryParam, Function, T> mapper) { + return this.query(index.getIndex(), queryParam, mapper); + } + + default Mono count(ElasticIndex index, QueryParam data) { + return this.count(index.getIndex(), data); + } + + default Mono commit(ElasticIndex index, T data) { + return this.commit(index.getIndex(), data); + } + + default Mono commit(ElasticIndex index, Collection data) { + return this.commit(index.getIndex(), data); + } + + default Mono commit(ElasticIndex index, Publisher data) { + return this.commit(index.getIndex(), data); + } - Mono commit(ElasticIndex index, Publisher data); } diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/IndexAliasOperationService.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/IndexAliasOperationService.java deleted file mode 100644 index 29af4072..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/IndexAliasOperationService.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.jetlinks.community.elastic.search.service; - -import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; -import reactor.core.publisher.Mono; - -/** - * @author bsetfeng - * @since 1.0 - **/ -public interface IndexAliasOperationService { - - Mono indexAliasIsExists(String index); - - Mono AddAlias(IndicesAliasesRequest request); -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/IndexOperationService.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/IndexOperationService.java deleted file mode 100644 index c5b65def..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/IndexOperationService.java +++ /dev/null @@ -1,18 +0,0 @@ -package org.jetlinks.community.elastic.search.service; - -import org.elasticsearch.client.indices.CreateIndexRequest; -import org.jetlinks.community.elastic.search.index.mapping.IndexMappingMetadata; -import reactor.core.publisher.Mono; - -/** - * @author bsetfeng - * @since 1.0 - **/ -public interface IndexOperationService { - - Mono indexIsExists(String index); - - Mono init(CreateIndexRequest request); - - Mono getIndexMappingMetadata(String index); -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/IndexTemplateOperationService.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/IndexTemplateOperationService.java deleted file mode 100644 index 867656b8..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/IndexTemplateOperationService.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.jetlinks.community.elastic.search.service; - -import org.elasticsearch.client.indices.PutIndexTemplateRequest; -import reactor.core.publisher.Mono; - -/** - * @author bsetfeng - * @since 1.0 - **/ -public interface IndexTemplateOperationService { - - Mono indexTemplateIsExists(String index); - - Mono putTemplate(PutIndexTemplateRequest request); -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/timeseries/AbstractTimeSeriesService.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/timeseries/AbstractTimeSeriesService.java deleted file mode 100644 index 3d594fe6..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/timeseries/AbstractTimeSeriesService.java +++ /dev/null @@ -1,20 +0,0 @@ -package org.jetlinks.community.elastic.search.timeseries; - -import org.hswebframework.ezorm.core.param.QueryParam; -import org.jetlinks.community.timeseries.TimeSeriesService; - -/** - * @author bsetfeng - * @since 1.0 - **/ -public abstract class AbstractTimeSeriesService implements TimeSeriesService { - - - protected QueryParam filterAddDefaultSort(QueryParam queryParam) { - - if (queryParam.getSorts().isEmpty()) { - queryParam.orderBy("timestamp").desc(); - } - return queryParam; - } -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/timeseries/ESAbstractTimeSeriesManager.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/timeseries/ESAbstractTimeSeriesManager.java deleted file mode 100644 index d99b66e5..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/timeseries/ESAbstractTimeSeriesManager.java +++ /dev/null @@ -1,77 +0,0 @@ -package org.jetlinks.community.elastic.search.timeseries; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.action.admin.indices.alias.Alias; -import org.jetlinks.community.elastic.search.index.IndexTemplateProvider; -import org.jetlinks.community.elastic.search.manager.StandardsIndexManager; -import org.jetlinks.community.elastic.search.service.IndexOperationService; -import org.jetlinks.community.elastic.search.service.IndexTemplateOperationService; -import org.jetlinks.community.timeseries.TimeSeriesManager; -import org.jetlinks.community.timeseries.TimeSeriesMetric; - -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -/** - * @author bsetfeng - * @since 1.0 - **/ -@Getter -@Slf4j -public abstract class ESAbstractTimeSeriesManager implements TimeSeriesManager { - - - private Map localMetricMap = new ConcurrentHashMap<>(); - - protected StandardsIndexManager standardsIndexManager; - - protected IndexOperationService indexOperationService; - - protected IndexTemplateOperationService indexTemplateOperationService; - - public ESAbstractTimeSeriesManager(IndexOperationService indexOperationService, - StandardsIndexManager standardsIndexManager, - IndexTemplateOperationService indexTemplateOperationService) { - this.indexOperationService = indexOperationService; - this.standardsIndexManager = standardsIndexManager; - this.indexTemplateOperationService = indexTemplateOperationService; - } - - - // TODO: 2020/2/11 策略变更动态更新实现 - protected LocalTimeSeriesMetric getLocalTimeSeriesMetric(TimeSeriesMetric metric) { - - return localMetricMap.computeIfAbsent(metric.getId().toLowerCase(), index -> { - String standardsIndex = getStandardsIndexManager().getStandardsIndex(index); - String templateName = IndexTemplateProvider.getIndexTemplate(index); - return new LocalTimeSeriesMetric( - getStandardsIndexManager().indexIsChange(index), - index, - standardsIndex, - IndexAliasProvider.getIndexAlias(index), - templateName, - Collections.singletonList(index.concat("*"))); - }); - } - - @Getter - @AllArgsConstructor - static class LocalTimeSeriesMetric { - - private boolean indexHasStrategy; - - private String index; - - private String standardsIndex; - - private Alias alias; - - private String templateName; - - private List indexTemplatePatterns; - } -} \ No newline at end of file diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/timeseries/ESAggregationData.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/timeseries/ESAggregationData.java deleted file mode 100644 index 55684ad1..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/timeseries/ESAggregationData.java +++ /dev/null @@ -1,24 +0,0 @@ -package org.jetlinks.community.elastic.search.timeseries; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import org.jetlinks.community.timeseries.query.AggregationData; - -import java.util.Map; - -/** - * @author bsetfeng - * @since 1.0 - **/ - -@Getter -@AllArgsConstructor -public class ESAggregationData implements AggregationData { - - private Map map; - - @Override - public Map asMap() { - return this.map; - } -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/timeseries/ESTimeSeriesManager.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/timeseries/ESTimeSeriesManager.java deleted file mode 100644 index 2ccb0eb0..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/timeseries/ESTimeSeriesManager.java +++ /dev/null @@ -1,100 +0,0 @@ -package org.jetlinks.community.elastic.search.timeseries; - -import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.client.indices.PutIndexTemplateRequest; -import org.jetlinks.core.metadata.DataType; -import org.jetlinks.core.metadata.PropertyMetadata; -import org.jetlinks.core.metadata.types.DateTimeType; -import org.jetlinks.community.elastic.search.enums.FieldDateFormat; -import org.jetlinks.community.elastic.search.enums.FieldType; -import org.jetlinks.community.elastic.search.index.CreateIndex; -import org.jetlinks.community.elastic.search.index.mapping.MappingFactory; -import org.jetlinks.community.elastic.search.manager.StandardsIndexManager; -import org.jetlinks.community.elastic.search.service.IndexOperationService; -import org.jetlinks.community.elastic.search.service.IndexTemplateOperationService; -import org.jetlinks.community.timeseries.TimeSeriesMetadata; -import org.jetlinks.community.timeseries.TimeSeriesMetric; -import org.jetlinks.community.timeseries.TimeSeriesService; -import org.springframework.stereotype.Service; -import reactor.core.publisher.Mono; - -import java.util.Collections; -import java.util.List; - -/** - * @author bsetfeng - * @since 1.0 - **/ -@Service -@Slf4j -public class ESTimeSeriesManager extends ESAbstractTimeSeriesManager { - - private final TimeSeriesServiceRegisterCenter timeSeriesServiceRegisterCenter; - - public ESTimeSeriesManager(TimeSeriesServiceRegisterCenter timeSeriesServiceRegisterCenter, - IndexOperationService indexOperationService, - StandardsIndexManager standardsIndexManager, - IndexTemplateOperationService indexTemplateOperationService) { - super(indexOperationService,standardsIndexManager,indexTemplateOperationService); - this.timeSeriesServiceRegisterCenter = timeSeriesServiceRegisterCenter; - - } - - @Override - public TimeSeriesService getService(TimeSeriesMetric metric) { - return timeSeriesServiceRegisterCenter.getTimeSeriesService(getLocalTimeSeriesMetric(metric)); - } - - @Override - public Mono registerMetadata(TimeSeriesMetadata metadata) { - LocalTimeSeriesMetric localTimeSeriesMetric = getLocalTimeSeriesMetric(metadata.getMetric()); - return registerIndexTemplate(localTimeSeriesMetric, metadata.getProperties()) - .doOnError(e -> log.error("初始化模板:{}失败", localTimeSeriesMetric.getTemplateName(), e)) - .then(); - } - - private Mono registerIndexTemplate(LocalTimeSeriesMetric localTimeSeriesMetric, List properties) { - return Mono.create(sink -> { - MappingFactory factory = CreateIndex.createInstance().createMapping(); - properties - .forEach(propertyMetadata -> addField(propertyMetadata, factory)); - addDefaultProperty(factory); - - PutIndexTemplateRequest request = new PutIndexTemplateRequest(localTimeSeriesMetric.getTemplateName()); - request.alias(localTimeSeriesMetric.getAlias()); - request.mapping(Collections.singletonMap("properties", factory.end().getMapping())); - request.patterns(localTimeSeriesMetric.getIndexTemplatePatterns()); - sink.success(request); - }) - .flatMap(indexTemplateOperationService::putTemplate) - .doOnError(e -> log.error("初始化template:{}失败", localTimeSeriesMetric.getTemplateName(), e)); - } - - - private void addDefaultProperty(MappingFactory factory) { - factory - .addFieldName("timestamp") - .addFieldType(FieldType.DATE) - .addFieldDateFormat(FieldDateFormat.epoch_millis, FieldDateFormat.simple_date, FieldDateFormat.strict_date_time) - .commit(); - } - - private void addField(PropertyMetadata property, MappingFactory factory) { - // TODO: 2020/2/4 对object 类型的支持 - DataType type = property.getValueType(); - if (type.getType().equalsIgnoreCase("date")) { - DateTimeType dateTimeType = (DateTimeType) type; - factory - .addFieldName(property.getId()) - .addFieldDateFormat(FieldDateFormat.epoch_millis, FieldDateFormat.simple_date, FieldDateFormat.strict_date_time) - //.addFieldDateFormat(dateTimeType.getFormat()) - .addFieldType(FieldType.DATE) - .commit(); - } else { - factory - .addFieldName(property.getId()) - .addFieldType(FieldType.ofJava(type.getType())) - .commit(); - } - } -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/timeseries/ESTimeSeriesService.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/timeseries/ESTimeSeriesService.java deleted file mode 100644 index 56804411..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/timeseries/ESTimeSeriesService.java +++ /dev/null @@ -1,241 +0,0 @@ -package org.jetlinks.community.elastic.search.timeseries; - -import lombok.AllArgsConstructor; -import org.elasticsearch.search.aggregations.bucket.histogram.ExtendedBounds; -import org.hswebframework.ezorm.core.param.QueryParam; -import org.hswebframework.ezorm.core.param.TermType; -import org.jetlinks.community.elastic.search.aggreation.bucket.Bucket; -import org.jetlinks.community.elastic.search.aggreation.bucket.BucketAggregationsStructure; -import org.jetlinks.community.elastic.search.aggreation.bucket.Sort; -import org.jetlinks.community.elastic.search.aggreation.enums.BucketType; -import org.jetlinks.community.elastic.search.aggreation.enums.MetricsType; -import org.jetlinks.community.elastic.search.aggreation.enums.OrderType; -import org.jetlinks.community.elastic.search.aggreation.metrics.MetricsAggregationStructure; -import org.jetlinks.community.elastic.search.index.ElasticIndex; -import org.jetlinks.community.elastic.search.service.AggregationService; -import org.jetlinks.community.elastic.search.service.ElasticSearchService; -import org.jetlinks.community.timeseries.TimeSeriesData; -import org.jetlinks.community.timeseries.query.AggregationData; -import org.jetlinks.community.timeseries.query.AggregationQueryParam; -import org.reactivestreams.Publisher; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; - -import java.time.Duration; -import java.util.*; -import java.util.stream.Collectors; - -/** - * @author bsetfeng - * @since 1.0 - **/ -@AllArgsConstructor -public class ESTimeSeriesService extends AbstractTimeSeriesService { - - private ESAbstractTimeSeriesManager.LocalTimeSeriesMetric localTimeSeriesMetric; - - private ElasticSearchService elasticSearchService; - - private AggregationService aggregationService; - - - @Override - public Flux query(QueryParam queryParam) { - return elasticSearchService.query( - cleverGetIndex(), - filterAddDefaultSort(queryParam), - Map.class) - .map(map -> new TimeSeriesData() { - - @Override - public long getTimestamp() { - return (long) map.get("timestamp"); - } - - @Override - public Map getData() { - return map; - } - }); - } - - @Override - public Mono count(QueryParam queryParam) { - return elasticSearchService.count( - cleverGetIndex(), - queryParam - ).map(Long::intValue); - } - - @Override - public Flux aggregation(AggregationQueryParam param) { - BucketAggregationsStructure structure = getStandardStructure(param); - //由于es一次性返回所有聚合查询结果，不完全支持响应式规范 - return aggregationService.bucketAggregation( - filterAddDefaultSort(addQueryTimeRange(param.getQueryParam(), param)), - structure, - cleverGetIndex()) - .onErrorResume(err -> Mono.empty()) - .flatMapMany(bucketResponse -> Flux.fromIterable(new BucketsParser(bucketResponse.getBuckets()).result) - .take(param.getLimit()) - .map(ESAggregationData::new)) - ; - } - - @Override - public Mono save(Publisher data) { - return Flux.from(data) - .map(timeSeriesData -> { - Map map = timeSeriesData.getData(); - map.put("timestamp", timeSeriesData.getTimestamp()); - return map; - }) - .as(stream -> elasticSearchService.commit(getDefaultIndex(), stream)); - } - - @Override - public Mono save(TimeSeriesData data) { - Map map = data.getData(); - map.put("timestamp", data.getTimestamp()); - return elasticSearchService.commit(getDefaultIndex(), map); - } - - - public void pack(BucketAggregationsStructure structureOutLayer, BucketAggregationsStructure structureInnerLayer) { - structureOutLayer.setSubBucketAggregation(Collections.singletonList(structureInnerLayer)); - } - - - protected BucketAggregationsStructure getStandardStructure(AggregationQueryParam param) { - List structures = new ArrayList<>(); - if (param.getGroupByTime() != null) { - structures.add(getDateTypeStructure(param)); - } - if (param.getGroupBy() != null && !param.getGroupBy().isEmpty()) { - structures.addAll(getTermTypeStructures(param)); - } - for (int i = 0; i < structures.size(); i++) { - if (i < structures.size() - 1) { - pack(structures.get(i), structures.get(i + 1)); - } - if (i == structures.size() - 1) { - structures.get(i).setSubMetricsAggregation(param.getAggColumns() - .stream() - .map(agg -> { - MetricsAggregationStructure metricsAggregationStructure = new MetricsAggregationStructure(); - metricsAggregationStructure.setField(agg.getProperty()); - metricsAggregationStructure.setName(agg.getAlias()); - metricsAggregationStructure.setType(MetricsType.of(agg.getAggregation().name())); - return metricsAggregationStructure; - }).collect(Collectors.toList())); - } - } - return structures.get(0); - } - - - protected BucketAggregationsStructure getDateTypeStructure(AggregationQueryParam param) { - BucketAggregationsStructure structure = new BucketAggregationsStructure(); - structure.setInterval(durationFormat(param.getGroupByTime().getInterval())); - structure.setType(BucketType.DATE_HISTOGRAM); - structure.setFormat(param.getGroupByTime().getFormat()); - structure.setName(param.getGroupByTime().getAlias()); - structure.setField("timestamp"); - structure.setSort(Sort.desc(OrderType.KEY)); - structure.setExtendedBounds(getExtendedBounds(param)); - return structure; - } - - protected List getTermTypeStructures(AggregationQueryParam param) { - return param.getGroupBy() - .stream() - .map(group -> { - BucketAggregationsStructure structure = new BucketAggregationsStructure(); - structure.setType(BucketType.TERMS); - structure.setSize(param.getLimit()); - structure.setField(group.getProperty()); - structure.setName(group.getAlias()); - return structure; - }).collect(Collectors.toList()); - } - - protected ElasticIndex getDefaultIndex() { - return ElasticIndex.createDefaultIndex(() -> localTimeSeriesMetric.getStandardsIndex(), () -> "_doc"); - } - - protected ElasticIndex cleverGetIndex() { - if (localTimeSeriesMetric.isIndexHasStrategy()) { - return ElasticIndex.createDefaultIndex(() -> localTimeSeriesMetric.getAlias().name(), () -> "_doc"); - } else { - return ElasticIndex.createDefaultIndex(() -> localTimeSeriesMetric.getIndex(), () -> "_doc"); - } - } - - - protected static QueryParam addQueryTimeRange(QueryParam queryParam, AggregationQueryParam param) { - queryParam.and( - "timestamp", - TermType.btw, - Arrays.asList(getStartWithParam(param), param.getEndWithTime())); - return queryParam; - } - - protected static ExtendedBounds getExtendedBounds(AggregationQueryParam param) { - return new ExtendedBounds(getStartWithParam(param), param.getEndWithTime()); - } - - private static long getStartWithParam(AggregationQueryParam param) { - long startWithParam = param.getStartWithTime(); - if (param.getGroupByTime() != null && param.getGroupByTime().getInterval() != null) { - long timeInterval = param.getGroupByTime().getInterval().toMillis() * param.getLimit(); - long tempStartWithParam = param.getEndWithTime() - timeInterval; - startWithParam = Math.max(tempStartWithParam, startWithParam); - } - return startWithParam; - } - - - protected static String durationFormat(Duration duration) { - String durationStr = duration.toString(); - if (durationStr.contains("S")) { - return duration.toMillis() / 1000 + "s"; - } else if (!durationStr.contains("S") && durationStr.contains("M")) { - return duration.toMinutes() + "m"; - } else if (!durationStr.contains("S") && !durationStr.contains("M")) { - if (duration.toHours() % 24 == 0) { - return duration.toDays() + "d"; - } else { - return duration.toHours() + "h"; - } - } - throw new UnsupportedOperationException("不支持的格式化，Duration: " + duration.toString()); - } - - static class BucketsParser { - - private List> result; - - public BucketsParser(List buckets) { - result = new ArrayList<>(); - buckets.forEach(bucket -> parser(bucket, new HashMap<>())); - } - - public void parser(Bucket bucket, Map fMap) { - addBucketProperty(bucket, fMap); - if (bucket.getBuckets() != null && !bucket.getBuckets().isEmpty()) { - bucket.getBuckets().forEach(b -> { - Map map = new HashMap<>(fMap); - addBucketProperty(b, map); - parser(b, map); - }); - } else { - result.add(fMap); - } - } - - private void addBucketProperty(Bucket bucket, Map fMap) { - fMap.put(bucket.getName(), bucket.getKey()); - fMap.putAll(bucket.toMap()); - } - } -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/timeseries/ElasticSearchTimeSeriesManager.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/timeseries/ElasticSearchTimeSeriesManager.java new file mode 100644 index 00000000..2545e12d --- /dev/null +++ b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/timeseries/ElasticSearchTimeSeriesManager.java @@ -0,0 +1,69 @@ +package org.jetlinks.community.elastic.search.timeseries; + +import lombok.extern.slf4j.Slf4j; +import org.jetlinks.core.metadata.SimplePropertyMetadata; +import org.jetlinks.core.metadata.types.DateTimeType; +import org.jetlinks.community.elastic.search.index.DefaultElasticSearchIndexMetadata; +import org.jetlinks.community.elastic.search.index.ElasticSearchIndexManager; +import org.jetlinks.community.elastic.search.service.AggregationService; +import org.jetlinks.community.elastic.search.service.ElasticSearchService; +import org.jetlinks.community.timeseries.TimeSeriesManager; +import org.jetlinks.community.timeseries.TimeSeriesMetadata; +import org.jetlinks.community.timeseries.TimeSeriesMetric; +import org.jetlinks.community.timeseries.TimeSeriesService; +import org.springframework.stereotype.Service; +import reactor.core.publisher.Mono; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * @author bsetfeng + * @author zhouhao + * @since 1.0 + **/ +@Service +@Slf4j +public class ElasticSearchTimeSeriesManager implements TimeSeriesManager { + + + private Map serviceMap = new ConcurrentHashMap<>(16); + + protected final ElasticSearchIndexManager indexManager; + + private final ElasticSearchService elasticSearchService; + + private final AggregationService aggregationService; + + + public ElasticSearchTimeSeriesManager(ElasticSearchIndexManager indexManager, + ElasticSearchService elasticSearchService, + AggregationService aggregationService) { + this.elasticSearchService = elasticSearchService; + this.indexManager = indexManager; + this.aggregationService = aggregationService; + } + + @Override + public TimeSeriesService getService(TimeSeriesMetric metric) { + return getService(metric.getId()); + } + + @Override + public TimeSeriesService getService(String metric) { + return serviceMap.computeIfAbsent(metric, + id -> new ElasticSearchTimeSeriesService(id, elasticSearchService, aggregationService)); + } + + + @Override + public Mono registerMetadata(TimeSeriesMetadata metadata) { + //默认字段 + SimplePropertyMetadata timestamp = new SimplePropertyMetadata(); + timestamp.setId("timestamp"); + timestamp.setValueType(new DateTimeType()); + return indexManager.putIndex(new DefaultElasticSearchIndexMetadata(metadata.getMetric().getId(), metadata.getProperties()) + .addProperty(timestamp)); + } + +} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/timeseries/ElasticSearchTimeSeriesService.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/timeseries/ElasticSearchTimeSeriesService.java new file mode 100644 index 00000000..c18505b4 --- /dev/null +++ b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/timeseries/ElasticSearchTimeSeriesService.java @@ -0,0 +1,68 @@ +package org.jetlinks.community.elastic.search.timeseries; + +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.hswebframework.ezorm.core.param.QueryParam; +import org.jetlinks.community.elastic.search.service.AggregationService; +import org.jetlinks.community.elastic.search.service.ElasticSearchService; +import org.jetlinks.community.timeseries.TimeSeriesData; +import org.jetlinks.community.timeseries.TimeSeriesService; +import org.jetlinks.community.timeseries.query.AggregationData; +import org.jetlinks.community.timeseries.query.AggregationQueryParam; +import org.reactivestreams.Publisher; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.util.Map; + +@AllArgsConstructor +@Slf4j +public class ElasticSearchTimeSeriesService implements TimeSeriesService { + + private String index; + + private ElasticSearchService elasticSearchService; + + private AggregationService aggregationService; + + @Override + public Flux query(QueryParam queryParam) { + return elasticSearchService.query(index, queryParam, map -> TimeSeriesData.of((Long) map.get("timestamp"), map)); + } + + @Override + public Mono count(QueryParam queryParam) { + return elasticSearchService + .count(index, queryParam) + .map(Long::intValue); + } + + @Override + public Flux aggregation(AggregationQueryParam queryParam) { + return aggregationService + .aggregation(index, queryParam) + .onErrorResume(err -> { + log.error(err.getMessage(), err); + return Mono.empty(); + }) + .map(AggregationData::of); + + } + + + @Override + public Mono save(Publisher data) { + return Flux.from(data) + .flatMap(this::save) + .then(); + } + + @Override + public Mono save(TimeSeriesData data) { + return Mono.defer(() -> { + Map mapData = data.getData(); + mapData.put("timestamp", data.getTimestamp()); + return elasticSearchService.commit(index, mapData); + }); + } +} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/timeseries/IndexAliasProvider.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/timeseries/IndexAliasProvider.java deleted file mode 100644 index a565f7fc..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/timeseries/IndexAliasProvider.java +++ /dev/null @@ -1,14 +0,0 @@ -package org.jetlinks.community.elastic.search.timeseries; - -import org.elasticsearch.action.admin.indices.alias.Alias; - -/** - * @author bsetfeng - * @since 1.0 - **/ -public interface IndexAliasProvider { - - static Alias getIndexAlias(String index) { - return new Alias(index.concat("_alias")); - } -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/timeseries/TimeSeriesServiceRegisterCenter.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/timeseries/TimeSeriesServiceRegisterCenter.java deleted file mode 100644 index fb16c037..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/timeseries/TimeSeriesServiceRegisterCenter.java +++ /dev/null @@ -1,32 +0,0 @@ -package org.jetlinks.community.elastic.search.timeseries; - -import org.jetlinks.community.elastic.search.service.AggregationService; -import org.jetlinks.community.elastic.search.service.ElasticSearchService; -import org.jetlinks.community.timeseries.TimeSeriesService; -import org.springframework.stereotype.Component; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -/** - * @author bsetfeng - * @since 1.0 - **/ -@Component -public class TimeSeriesServiceRegisterCenter { - - private final ElasticSearchService elasticSearchService; - - private final AggregationService aggregationService; - - private Map serviceMap = new ConcurrentHashMap<>(16); - - public TimeSeriesServiceRegisterCenter(AggregationService aggregationService, ElasticSearchService elasticSearchService) { - this.aggregationService = aggregationService; - this.elasticSearchService = elasticSearchService; - } - - public TimeSeriesService getTimeSeriesService(ESAbstractTimeSeriesManager.LocalTimeSeriesMetric metric) { - return serviceMap.computeIfAbsent(metric.getIndex(), i -> new ESTimeSeriesService(metric, elasticSearchService, aggregationService)); - } -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/translate/QueryParamTranslator.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/translate/QueryParamTranslator.java deleted file mode 100644 index 3756dded..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/translate/QueryParamTranslator.java +++ /dev/null @@ -1,44 +0,0 @@ -package org.jetlinks.community.elastic.search.translate; - -import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.hswebframework.ezorm.core.param.QueryParam; -import org.jetlinks.community.elastic.search.enums.LinkTypeEnum; - -import java.util.Objects; - -/** - * @author bsetfeng - * @since 1.0 - **/ -@Slf4j -public class QueryParamTranslator { - - public static QueryBuilder translate(QueryParam queryParam) { - - BoolQueryBuilder query = QueryBuilders.boolQuery(); - Objects.requireNonNull(queryParam, "QueryParam must not null.") - .getTerms() - .forEach(term -> LinkTypeEnum.of(term.getType().name()) - .ifPresent(e -> e.process(query, term))); - return query; - } - -// public static SearchSourceBuilder transSourceBuilder(QueryParam queryParam) { -// SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); -// if (queryParam.isPaging()) { -// sourceBuilder.from(queryParam.getPageIndex() * queryParam.getPageSize()); -// sourceBuilder.size(queryParam.getPageSize()); -// } -// queryParam.getSorts() -// .forEach(sort -> { -// if (!StringUtils.isEmpty(sort.getName())) { -// sourceBuilder.sort(sort.getName(), SortOrder.fromString(sort.getOrder())); -// } -// -// }); -// return sourceBuilder.query(translate(queryParam)); -// } -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/utils/DateTimeUtils.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/utils/DateTimeUtils.java deleted file mode 100644 index 83521368..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/utils/DateTimeUtils.java +++ /dev/null @@ -1,54 +0,0 @@ -package org.jetlinks.community.elastic.search.utils; - -import com.alibaba.fastjson.JSON; -import lombok.extern.slf4j.Slf4j; - -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.List; - -/** - * @author bsetfeng - * @since 1.0 - **/ -@Slf4j -public class DateTimeUtils { - - public static Object formatDateToTimestamp(Object date, List formats) { - return TermCommonUtils.getStandardsTermValue( - formatDateArrayToTimestamp(TermCommonUtils.convertToList(date), formats) - ); - } - - private static Object formatDateStringToTimestamp(String dateString, List formats) { - for (String format : formats) { - try { - return formatDateStringToTimestamp(dateString, format); - } catch (Exception e) { - log.debug("按格式:{}解析时间字符串:{}错误", format, dateString); - } - } - throw new UnsupportedOperationException("不支持的时间转换" + "formats:" + - JSON.toJSONString(formats) + "dateString:" + dateString); - } - - private static long formatDateStringToTimestamp(String dateString, String format) { - DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern(format); - LocalDateTime dateTime = LocalDateTime.parse(dateString, dateTimeFormatter); - return dateTime.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); - } - - private static List formatDateArrayToTimestamp(List values, List formats) { - List result = new ArrayList<>(); - for (Object value : values) { - if (value instanceof String) { - result.add(formatDateStringToTimestamp(value.toString(), formats)); - } else { - result.add(value); - } - } - return result; - } -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/utils/ElasticSearchConverter.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/utils/ElasticSearchConverter.java new file mode 100644 index 00000000..23aeba74 --- /dev/null +++ b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/utils/ElasticSearchConverter.java @@ -0,0 +1,63 @@ +package org.jetlinks.community.elastic.search.utils; + +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.hswebframework.ezorm.core.param.QueryParam; +import org.jetlinks.core.metadata.Converter; +import org.jetlinks.core.metadata.DataType; +import org.jetlinks.core.metadata.PropertyMetadata; +import org.jetlinks.core.metadata.types.DateTimeType; +import org.jetlinks.core.metadata.types.GeoPoint; +import org.jetlinks.core.metadata.types.GeoType; +import org.jetlinks.community.elastic.search.index.ElasticSearchIndexMetadata; + +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class ElasticSearchConverter { + + + public static SearchSourceBuilder convertSearchSourceBuilder(QueryParam queryParam, ElasticSearchIndexMetadata metadata) { + return QueryParamTranslator.convertSearchSourceBuilder(queryParam, metadata); + } + + public static Map convertDataToElastic(Map data, List properties) { + for (PropertyMetadata property : properties) { + DataType type = property.getValueType(); + Object val = data.get(property.getId()); + if (val == null) { + continue; + } + //处理地理位置类型 + if (type instanceof GeoType) { + GeoPoint point = ((GeoType) type).convert(val); + Map geoData = new HashMap<>(); + geoData.put("lat", point.getLat()); + geoData.put("lon", point.getLon()); + data.put(property.getId(), geoData); + }if (type instanceof DateTimeType) { + Date point = ((DateTimeType) type).convert(val); + data.put(property.getId(), point.getTime()); + } else if (type instanceof Converter) { + data.put(property.getId(), ((Converter) type).convert(val)); + } + } + return data; + } + + public static Map convertDataFromElastic(Map data, List properties) { + for (PropertyMetadata property : properties) { + DataType type = property.getValueType(); + Object val = data.get(property.getId()); + if (val == null) { + continue; + } + //处理地理位置类型 + if (type instanceof GeoType) { + data.put(property.getId(), ((GeoType) type).convertToMap(val)); + } + } + return data; + } +} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/utils/QueryParamTranslator.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/utils/QueryParamTranslator.java new file mode 100644 index 00000000..91be9df7 --- /dev/null +++ b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/utils/QueryParamTranslator.java @@ -0,0 +1,81 @@ +package org.jetlinks.community.elastic.search.utils; + +import lombok.extern.slf4j.Slf4j; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.sort.SortOrder; +import org.hswebframework.ezorm.core.param.QueryParam; +import org.hswebframework.ezorm.core.param.Sort; +import org.hswebframework.ezorm.core.param.Term; +import org.jetlinks.core.metadata.DataType; +import org.jetlinks.core.metadata.PropertyMetadata; +import org.jetlinks.core.metadata.types.GeoType; +import org.jetlinks.community.elastic.search.index.ElasticSearchIndexMetadata; +import org.jetlinks.community.elastic.search.parser.DefaultLinkTypeParser; +import org.springframework.util.StringUtils; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.BiConsumer; +import java.util.function.Consumer; + +/** + * @author zhouhao + * @since 1.0 + **/ +@Slf4j +public class QueryParamTranslator { + + static DefaultLinkTypeParser linkTypeParser = new DefaultLinkTypeParser(); + + static Consumer doNotingParamConverter = (term -> { + }); + + static Map> converter = new ConcurrentHashMap<>(); + + static BiConsumer defaultDataTypeConverter = (type, term) -> { + + }; + + static { + + //地理位置查询 + converter.put(GeoType.ID, (type, term) -> { + // TODO: 2020/3/5 + + }); + } + + public static SearchSourceBuilder convertSearchSourceBuilder(QueryParam queryParam, ElasticSearchIndexMetadata metadata) { + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + if (queryParam.isPaging()) { + sourceBuilder.from(queryParam.getPageIndex() * queryParam.getPageSize()); + sourceBuilder.size(queryParam.getPageSize()); + } + for (Sort sort : queryParam.getSorts()) { + if (!StringUtils.isEmpty(sort.getName())) { + sourceBuilder.sort(sort.getName(), SortOrder.fromString(sort.getOrder())); + } + } + BoolQueryBuilder queryBuilders = QueryBuilders.boolQuery(); + Consumer paramConverter = doNotingParamConverter; + if (metadata != null) { + paramConverter = t -> { + if (StringUtils.isEmpty(t.getColumn())) { + return; + } + PropertyMetadata property = metadata.getProperty(t.getColumn()); + if (null != property) { + DataType type = property.getValueType(); + converter.getOrDefault(type.getId(), defaultDataTypeConverter).accept(type, t); + } + }; + } + for (Term term : queryParam.getTerms()) { + linkTypeParser.process(term, paramConverter, queryBuilders); + } + return sourceBuilder.query(queryBuilders); + } + +} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/utils/ReactorActionListener.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/utils/ReactorActionListener.java new file mode 100644 index 00000000..dc454c8e --- /dev/null +++ b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/utils/ReactorActionListener.java @@ -0,0 +1,56 @@ +package org.jetlinks.community.elastic.search.utils; + +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.action.ActionListener; +import org.hswebframework.web.exception.BusinessException; +import reactor.core.publisher.Mono; + +import java.util.function.Consumer; +import java.util.function.Function; + + +public class ReactorActionListener { + + public static Mono mono(Consumer> listenerConsumer, + Function> onSuccess, + Function> onError) { + return Mono.>create(sink -> { + listenerConsumer.accept(new ActionListener() { + @Override + public void onResponse(T t) { + try { + sink.success(onSuccess.apply(t)); + } catch (Exception e) { + sink.error(e); + } + } + + @Override + public void onFailure(Exception e) { + try { + sink.success(onError.apply(e)); + } catch (Exception e2) { + sink.error(e2); + } + } + }); + + }).flatMap(Function.identity()) + .onErrorResume(ElasticsearchStatusException.class, e -> { + if (e.status().getStatus() == 404) { + return Mono.empty(); + } + return Mono.error(new BusinessException(e.getMessage(), e)); + }); + } + + + public static Mono mono(Consumer> listenerConsumer, + Function> onSuccess) { + return mono(listenerConsumer, onSuccess, Mono::error); + } + + public static Mono mono(Consumer> listenerConsumer) { + return mono(listenerConsumer, Mono::justOrEmpty, Mono::error); + } +} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/utils/TermCommonUtils.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/utils/TermCommonUtils.java index a94599aa..767670b8 100644 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/utils/TermCommonUtils.java +++ b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/utils/TermCommonUtils.java @@ -24,7 +24,7 @@ public class TermCommonUtils { return new ArrayList(((Collection) value)); } - return Arrays.asList(value); + return Collections.singletonList(value); } public static Object getStandardsTermValue(List value) { diff --git a/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/configuration/LoggingConfiguration.java b/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/configuration/LoggingConfiguration.java index 15951a80..44d985f4 100644 --- a/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/configuration/LoggingConfiguration.java +++ b/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/configuration/LoggingConfiguration.java @@ -1,19 +1,11 @@ package org.jetlinks.community.logging.configuration; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.client.indices.CreateIndexRequest; -import org.jetlinks.community.elastic.search.enums.FieldDateFormat; -import org.jetlinks.community.elastic.search.enums.FieldType; -import org.jetlinks.community.elastic.search.index.CreateIndex; -import org.jetlinks.community.elastic.search.service.IndexOperationService; -import org.jetlinks.community.logging.event.handler.LoggerIndexProvider; import org.jetlinks.community.logging.logback.SystemLoggingAppender; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.ApplicationEventPublisherAware; import org.springframework.context.annotation.Configuration; -import org.springframework.web.bind.annotation.PostMapping; @Configuration @EnableConfigurationProperties(LoggingProperties.class) diff --git a/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/event/handler/AccessLoggerEventHandler.java b/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/event/handler/AccessLoggerEventHandler.java index 72bd9891..9137aed4 100644 --- a/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/event/handler/AccessLoggerEventHandler.java +++ b/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/event/handler/AccessLoggerEventHandler.java @@ -1,12 +1,12 @@ package org.jetlinks.community.logging.event.handler; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.client.indices.CreateIndexRequest; -import org.jetlinks.community.elastic.search.enums.FieldDateFormat; -import org.jetlinks.community.elastic.search.enums.FieldType; -import org.jetlinks.community.elastic.search.index.CreateIndex; +import org.jetlinks.core.metadata.types.DateTimeType; +import org.jetlinks.core.metadata.types.ObjectType; +import org.jetlinks.core.metadata.types.StringType; +import org.jetlinks.community.elastic.search.index.DefaultElasticSearchIndexMetadata; +import org.jetlinks.community.elastic.search.index.ElasticSearchIndexManager; import org.jetlinks.community.elastic.search.service.ElasticSearchService; -import org.jetlinks.community.elastic.search.service.IndexOperationService; import org.jetlinks.community.logging.access.SerializableAccessLog; import org.springframework.context.event.EventListener; import org.springframework.core.annotation.Order; @@ -24,23 +24,23 @@ public class AccessLoggerEventHandler { private final ElasticSearchService elasticSearchService; - public AccessLoggerEventHandler(ElasticSearchService elasticSearchService, IndexOperationService indexOperationService) { + + public AccessLoggerEventHandler(ElasticSearchService elasticSearchService, ElasticSearchIndexManager indexManager) { this.elasticSearchService = elasticSearchService; - CreateIndexRequest accessLoggerIndex = CreateIndex.createInstance() - .addIndex(LoggerIndexProvider.ACCESS.getStandardIndex()) - .createMapping() - .addFieldName("requestTime").addFieldType(FieldType.DATE).addFieldDateFormat(FieldDateFormat.epoch_millis, FieldDateFormat.simple_date, FieldDateFormat.strict_date_time).commit() - .addFieldName("responseTime").addFieldType(FieldType.DATE).addFieldDateFormat(FieldDateFormat.epoch_millis, FieldDateFormat.simple_date, FieldDateFormat.strict_date_time).commit() - .addFieldName("action").addFieldType(FieldType.KEYWORD).commit() - .addFieldName("ip").addFieldType(FieldType.KEYWORD).commit() - .addFieldName("url").addFieldType(FieldType.KEYWORD).commit() - .addFieldName("httpHeaders").addFieldType(FieldType.OBJECT).commit() - .addFieldName("context").addFieldType(FieldType.OBJECT).commit() - .end() - .createIndexRequest(); - indexOperationService.init(accessLoggerIndex) - .doOnError(err -> log.error(err.getMessage(), err)) - .subscribe(); + indexManager.putIndex( + new DefaultElasticSearchIndexMetadata(LoggerIndexProvider.ACCESS.getIndex()) + .addProperty("requestTime", new DateTimeType()) + .addProperty("responseTime", new DateTimeType()) + .addProperty("action", new StringType()) + .addProperty("ip", new StringType()) + .addProperty("url", new StringType()) + .addProperty("httpHeaders", new ObjectType()) + .addProperty("context", new ObjectType() + .addProperty("userId",new StringType()) + .addProperty("username",new StringType()) + ) + ).subscribe(); + } diff --git a/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/event/handler/LoggerIndexProvider.java b/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/event/handler/LoggerIndexProvider.java index 9e27a152..af454a8c 100644 --- a/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/event/handler/LoggerIndexProvider.java +++ b/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/event/handler/LoggerIndexProvider.java @@ -12,10 +12,8 @@ import org.jetlinks.community.elastic.search.index.ElasticIndex; @AllArgsConstructor public enum LoggerIndexProvider implements ElasticIndex { - ACCESS("access_log", "_doc"), - SYSTEM("system_log", "_doc"); + ACCESS("access_logger"), + SYSTEM("system_logger"); private String index; - - private String type; } diff --git a/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/event/handler/SystemLoggerEventHandler.java b/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/event/handler/SystemLoggerEventHandler.java index 73aea0bf..8cdb4b9d 100644 --- a/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/event/handler/SystemLoggerEventHandler.java +++ b/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/event/handler/SystemLoggerEventHandler.java @@ -1,12 +1,12 @@ package org.jetlinks.community.logging.event.handler; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.client.indices.CreateIndexRequest; -import org.jetlinks.community.elastic.search.enums.FieldDateFormat; -import org.jetlinks.community.elastic.search.enums.FieldType; -import org.jetlinks.community.elastic.search.index.CreateIndex; +import org.jetlinks.core.metadata.types.DateTimeType; +import org.jetlinks.core.metadata.types.ObjectType; +import org.jetlinks.core.metadata.types.StringType; +import org.jetlinks.community.elastic.search.index.DefaultElasticSearchIndexMetadata; +import org.jetlinks.community.elastic.search.index.ElasticSearchIndexManager; import org.jetlinks.community.elastic.search.service.ElasticSearchService; -import org.jetlinks.community.elastic.search.service.IndexOperationService; import org.jetlinks.community.logging.system.SerializableSystemLog; import org.springframework.context.event.EventListener; import org.springframework.core.annotation.Order; @@ -25,22 +25,19 @@ public class SystemLoggerEventHandler { private final ElasticSearchService elasticSearchService; - public SystemLoggerEventHandler(ElasticSearchService elasticSearchService, IndexOperationService indexOperationService) { + public SystemLoggerEventHandler(ElasticSearchService elasticSearchService, ElasticSearchIndexManager indexManager) { this.elasticSearchService = elasticSearchService; - CreateIndexRequest systemLoggerIndex = CreateIndex.createInstance() - .addIndex(LoggerIndexProvider.SYSTEM.getStandardIndex()) - .createMapping() - .addFieldName("createTime").addFieldType(FieldType.DATE).addFieldDateFormat(FieldDateFormat.epoch_millis, FieldDateFormat.simple_date, FieldDateFormat.strict_date_time).commit() - .addFieldName("name").addFieldType(FieldType.KEYWORD).commit() - .addFieldName("level").addFieldType(FieldType.KEYWORD).commit() - .addFieldName("message").addFieldType(FieldType.KEYWORD).commit() - .end() - .createIndexRequest(); - - indexOperationService.init(systemLoggerIndex) - .doOnError(err -> log.error(err.getMessage(), err)) - .subscribe(); + indexManager.putIndex( + new DefaultElasticSearchIndexMetadata(LoggerIndexProvider.SYSTEM.getIndex()) + .addProperty("createTime", new DateTimeType()) + .addProperty("name", new StringType()) + .addProperty("level", new StringType()) + .addProperty("message", new StringType()) + .addProperty("context", new ObjectType() + .addProperty("requestId",new StringType()) + .addProperty("server",new StringType())) + ).subscribe(); } @EventListener diff --git a/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/configuration/RuleEngineLogIndexInitialize.java b/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/configuration/RuleEngineLogIndexInitialize.java index fe1d9b32..6c6ddaa0 100644 --- a/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/configuration/RuleEngineLogIndexInitialize.java +++ b/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/configuration/RuleEngineLogIndexInitialize.java @@ -1,13 +1,11 @@ package org.jetlinks.community.rule.engine.configuration; import lombok.extern.slf4j.Slf4j; -import org.jetlinks.community.elastic.search.enums.FieldDateFormat; -import org.jetlinks.community.elastic.search.enums.FieldType; -import org.jetlinks.community.elastic.search.index.CreateIndex; -import org.jetlinks.community.elastic.search.service.IndexOperationService; +import org.jetlinks.community.elastic.search.index.DefaultElasticSearchIndexMetadata; +import org.jetlinks.community.elastic.search.index.ElasticSearchIndexManager; import org.jetlinks.community.rule.engine.event.handler.RuleEngineLoggerIndexProvider; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.CommandLineRunner; +import org.jetlinks.core.metadata.types.DateTimeType; +import org.jetlinks.core.metadata.types.StringType; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; @@ -18,40 +16,23 @@ import org.springframework.stereotype.Component; @Component @Order(1) @Slf4j -public class RuleEngineLogIndexInitialize implements CommandLineRunner { +public class RuleEngineLogIndexInitialize { - private final IndexOperationService indexOperationService; - - @Autowired - public RuleEngineLogIndexInitialize(IndexOperationService indexOperationService) { - this.indexOperationService = indexOperationService; - } - - - @Override - public void run(String... args) throws Exception { - indexOperationService.init(CreateIndex.createInstance() - .addIndex(RuleEngineLoggerIndexProvider.RULE_LOG.getStandardIndex()) - .createMapping() - .addFieldName("createTime").addFieldType(FieldType.DATE).addFieldDateFormat(FieldDateFormat.epoch_millis, FieldDateFormat.simple_date, FieldDateFormat.strict_date_time).commit() - .addFieldName("level").addFieldType(FieldType.KEYWORD).commit() - .addFieldName("message").addFieldType(FieldType.KEYWORD).commit() - .addFieldName("nodeId").addFieldType(FieldType.KEYWORD).commit() - .addFieldName("instanceId").addFieldType(FieldType.KEYWORD).commit() - .end() - .createIndexRequest()) - .and( - indexOperationService.init(CreateIndex.createInstance() - .addIndex(RuleEngineLoggerIndexProvider.RULE_EVENT_LOG.getStandardIndex()) - .createMapping() - .addFieldName("createTime").addFieldType(FieldType.DATE).addFieldDateFormat(FieldDateFormat.epoch_millis, FieldDateFormat.simple_date, FieldDateFormat.strict_date_time).commit() - .addFieldName("event").addFieldType(FieldType.KEYWORD).commit() - .addFieldName("nodeId").addFieldType(FieldType.KEYWORD).commit() - .addFieldName("instanceId").addFieldType(FieldType.KEYWORD).commit() - .end() - .createIndexRequest()) + public RuleEngineLogIndexInitialize(ElasticSearchIndexManager indexManager) { + indexManager.putIndex(new DefaultElasticSearchIndexMetadata(RuleEngineLoggerIndexProvider.RULE_LOG.getIndex()) + .addProperty("createTime", new DateTimeType()) + .addProperty("level", new StringType()) + .addProperty("message", new StringType()) + .addProperty("nodeId", new StringType()) + .addProperty("instanceId", new StringType())) + .then( + indexManager.putIndex(new DefaultElasticSearchIndexMetadata(RuleEngineLoggerIndexProvider.RULE_LOG.getIndex()) + .addProperty("createTime", new DateTimeType()) + .addProperty("event", new StringType()) + .addProperty("nodeId", new StringType()) + .addProperty("instanceId", new StringType())) ) - .doOnError(err -> log.error(err.getMessage(), err)) .subscribe(); } + } diff --git a/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/event/handler/RuleLogHandler.java b/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/event/handler/RuleLogHandler.java index 5e526648..bfa00bcb 100644 --- a/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/event/handler/RuleLogHandler.java +++ b/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/event/handler/RuleLogHandler.java @@ -12,7 +12,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.event.EventListener; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; -import reactor.core.publisher.Mono; @Component @Slf4j @@ -26,7 +25,7 @@ public class RuleLogHandler { @EventListener public void handleRuleLog(LogInfo event) { RuleEngineExecuteLogInfo logInfo = FastBeanCopier.copy(event, new RuleEngineExecuteLogInfo()); - elasticSearchService.commit(RuleEngineLoggerIndexProvider.RULE_LOG, Mono.just(logInfo)) + elasticSearchService.commit(RuleEngineLoggerIndexProvider.RULE_LOG, logInfo) .subscribe(); } @@ -36,7 +35,7 @@ public class RuleLogHandler { if (!RuleEvent.NODE_EXECUTE_BEFORE.equals(event.getEvent()) && !RuleEvent.NODE_EXECUTE_RESULT.equals(event.getEvent())) { RuleEngineExecuteEventInfo eventInfo = FastBeanCopier.copy(event, new RuleEngineExecuteEventInfo()); - elasticSearchService.commit(RuleEngineLoggerIndexProvider.RULE_EVENT_LOG, Mono.just(eventInfo)) + elasticSearchService.commit(RuleEngineLoggerIndexProvider.RULE_EVENT_LOG, eventInfo) .subscribe(); } } diff --git a/jetlinks-components/timeseries-component/src/main/java/org/jetlinks/community/timeseries/TimeSeriesManager.java b/jetlinks-components/timeseries-component/src/main/java/org/jetlinks/community/timeseries/TimeSeriesManager.java index 9b874b56..3c2d196d 100644 --- a/jetlinks-components/timeseries-component/src/main/java/org/jetlinks/community/timeseries/TimeSeriesManager.java +++ b/jetlinks-components/timeseries-component/src/main/java/org/jetlinks/community/timeseries/TimeSeriesManager.java @@ -18,6 +18,8 @@ public interface TimeSeriesManager { */ TimeSeriesService getService(TimeSeriesMetric metric); + TimeSeriesService getService(String metric); + /** * 注册元数据 * diff --git a/jetlinks-components/timeseries-component/src/main/java/org/jetlinks/community/timeseries/query/AggregationData.java b/jetlinks-components/timeseries-component/src/main/java/org/jetlinks/community/timeseries/query/AggregationData.java index ee69e99e..7b9ecd57 100644 --- a/jetlinks-components/timeseries-component/src/main/java/org/jetlinks/community/timeseries/query/AggregationData.java +++ b/jetlinks-components/timeseries-component/src/main/java/org/jetlinks/community/timeseries/query/AggregationData.java @@ -14,4 +14,7 @@ public interface AggregationData extends ValueObject { return Optional.ofNullable(asMap().get(name)); } + static AggregationData of(Map map){ + return ()->map; + } } diff --git a/jetlinks-components/timeseries-component/src/main/java/org/jetlinks/community/timeseries/query/AggregationQueryParam.java b/jetlinks-components/timeseries-component/src/main/java/org/jetlinks/community/timeseries/query/AggregationQueryParam.java index 1fa1ed59..080c37b1 100644 --- a/jetlinks-components/timeseries-component/src/main/java/org/jetlinks/community/timeseries/query/AggregationQueryParam.java +++ b/jetlinks-components/timeseries-component/src/main/java/org/jetlinks/community/timeseries/query/AggregationQueryParam.java @@ -6,12 +6,9 @@ import org.hswebframework.ezorm.core.dsl.Query; import org.hswebframework.ezorm.core.param.QueryParam; import java.time.Duration; -import java.time.LocalDateTime; -import java.time.ZoneId; import java.util.ArrayList; import java.util.Date; import java.util.List; -import java.util.Optional; import java.util.function.Consumer; import java.util.function.Function; @@ -35,6 +32,8 @@ public class AggregationQueryParam { private long endWithTime = System.currentTimeMillis(); + private String timeProperty = "timestamp"; + //条件过滤 private QueryParam queryParam = new QueryParam(); diff --git a/jetlinks-components/timeseries-component/src/main/java/org/jetlinks/community/timeseries/query/TimeGroup.java b/jetlinks-components/timeseries-component/src/main/java/org/jetlinks/community/timeseries/query/TimeGroup.java index ba5c851c..75093a43 100644 --- a/jetlinks-components/timeseries-component/src/main/java/org/jetlinks/community/timeseries/query/TimeGroup.java +++ b/jetlinks-components/timeseries-component/src/main/java/org/jetlinks/community/timeseries/query/TimeGroup.java @@ -13,6 +13,8 @@ import java.time.Duration; @NoArgsConstructor public class TimeGroup { + private String property; + //时间分组间隔,如: 1d , 30s private Duration interval; @@ -23,5 +25,9 @@ public class TimeGroup { */ private String format; - + public TimeGroup(Duration interval, String alias, String format) { + this.interval = interval; + this.alias = alias; + this.format = format; + } } diff --git a/jetlinks-standalone/src/main/resources/application.yml b/jetlinks-standalone/src/main/resources/application.yml index 5aaadba2..baee0f9c 100644 --- a/jetlinks-standalone/src/main/resources/application.yml +++ b/jetlinks-standalone/src/main/resources/application.yml @@ -40,6 +40,8 @@ elasticsearch: connect-timeout: 5000 socket-timeout: 5000 connection-request-timeout: 8000 + index: + default-strategy: time-by-month #默认es的索引按月进行分表, direct则为直接操作索引. device: message: writer: