From 587c4f4d484db23a70b3a3104f45b4b74f83cd4f Mon Sep 17 00:00:00 2001 From: zhouhao Date: Sun, 27 Apr 2025 17:28:32 +0800 Subject: [PATCH] =?UTF-8?q?refactor:=20spring-boot3=20=E9=80=82=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 3 +- .run/JetLinksApplication.run.xml | 18 + README.md | 6 +- docker/run-all/docker-compose.yml | 4 +- jetlinks-components/common-component/pom.xml | 13 +- .../jetlinks/community/PropertyConstants.java | 9 +- .../jetlinks/community/PropertyMetric.java | 2 +- .../org/jetlinks/community/TimerSpec.java | 4 +- .../configuration/CommonConfiguration.java | 29 +- .../dictionary/DictionaryConfiguration.java | 15 +- .../dictionary/DictionaryEventHandler.java | 32 +- .../community/reactorql/EnableReactorQL.java | 26 - .../community/reactorql/ReactorQL.java | 37 - .../ReactorQLBeanDefinitionRegistrar.java | 59 - .../reactorql/ReactorQLFactoryBean.java | 164 -- .../reactorql/ReactorQLOperation.java | 22 - .../community/reactorql/term/TermValue.java | 6 +- ...ot.autoconfigure.AutoConfiguration.imports | 3 +- .../configure-component/pom.xml | 4 +- .../JetLinksCommonConfiguration.java | 5 + .../compatible/PathCompatibleFilter.java | 34 + .../device/DeviceClusterConfiguration.java | 36 +- .../configure/trace/TraceWebFilter.java | 10 +- .../dashboard-component/pom.xml | 2 +- .../datasource-component/pom.xml | 74 + .../datasource/AbstractDataSource.java | 92 + .../community/datasource/DataSource.java | 75 + .../datasource/DataSourceConfig.java | 22 + .../datasource/DataSourceConfigManager.java | 36 + .../datasource/DataSourceConstants.java | 46 + .../datasource/DataSourceManager.java | 84 + .../datasource/DataSourceProvider.java | 101 ++ .../community/datasource/DataSourceState.java | 38 + .../community/datasource/DataSourceType.java | 7 + .../datasource/DefaultDataSourceManager.java | 235 +++ .../command/CommandHandlerProvider.java | 16 + .../command/DataSourceCommandConfig.java | 48 + .../DataSourceCommandSupportManager.java | 238 +++ ...ataSourceHandlerProviderConfiguration.java | 14 + .../DataSourceHandlerProviderRegister.java | 31 + .../DataSourceManagerConfiguration.java | 24 + .../DataSourceNotExistException.java | 15 + .../datasource/rdb/DefaultRDBDataSource.java | 345 ++++ .../datasource/rdb/RDBDataSource.java | 33 + .../rdb/RDBDataSourceProperties.java | 93 + .../datasource/rdb/RDBDataSourceProvider.java | 217 +++ .../datasource/rdb/RDBDataSourceType.java | 18 + .../rdb/RDBJdbcReactiveSqlExecutor.java | 58 + .../rdb/RDBJdbcSyncSqlExecutor.java | 26 + .../datasource/rdb/command/Column.java | 39 + .../datasource/rdb/command/Count.java | 69 + .../rdb/command/CreateOrAlterTable.java | 39 + .../datasource/rdb/command/DropColumn.java | 27 + .../datasource/rdb/command/ExecuteSql.java | 206 +++ .../datasource/rdb/command/GetTable.java | 24 + .../datasource/rdb/command/GetTables.java | 29 + .../datasource/rdb/command/QueryList.java | 77 + .../datasource/rdb/command/QueryPager.java | 121 ++ .../datasource/rdb/command/RDBCommand.java | 9 + .../rdb/command/RDBRequestListCommand.java | 65 + .../rdb/command/RDBRequestPagerCommand.java | 58 + .../datasource/rdb/command/Refresh.java | 30 + .../datasource/rdb/command/Table.java | 27 + .../datasource/rdb/command/Upsert.java | 39 + ...ot.autoconfigure.AutoConfiguration.imports | 2 + .../i18n/datasource/messages_en.properties | 3 + .../i18n/datasource/messages_zh.properties | 3 + .../elasticsearch-component/.gitignore | 1 + .../elasticsearch-7x/pom.xml | 42 + .../search/ElasticSearch7xSupport.java | 67 + .../search/enums/ElasticSearch7xTermType.java | 232 +++ ...munity.elastic.search.ElasticSearchSupport | 1 + .../elasticsearch-8x/pom.xml | 40 + .../search/ElasticSearch8xSupport.java | 73 + .../search/enums/ElasticSearch8xTermType.java | 238 +++ ...munity.elastic.search.ElasticSearchSupport | 1 + .../elasticsearch-core/pom.xml | 129 ++ .../elastic/search/ElasticSearchSupport.java | 111 ++ .../ElasticSearchClientConfiguration.java | 44 + .../ElasticSearchConfiguration.java | 101 +- .../ElasticSearchProperties.java | 6 + .../ElasticSearchThingDataConfiguration.java | 3 + .../search/enums/ElasticDateFormat.java | 13 +- .../search/enums/ElasticPropertyType.java | 21 +- .../search/enums/ElasticSearchTermType.java | 26 + .../search/enums/ElasticSearchTermTypes.java | 41 + .../DefaultElasticSearchIndexManager.java | 20 +- .../DefaultElasticSearchIndexMetadata.java | 12 +- .../elastic/search/index/ElasticIndex.java | 0 .../index/ElasticSearchIndexCustomizer.java | 7 + .../index/ElasticSearchIndexManager.java | 15 +- .../index/ElasticSearchIndexMetadata.java | 2 +- .../index/ElasticSearchIndexProperties.java | 58 + .../index/ElasticSearchIndexStrategy.java | 5 + .../AbstractElasticSearchIndexStrategy.java | 277 +++ .../AffixesElasticSearchIndexStrategy.java | 86 + .../DirectElasticSearchIndexStrategy.java | 0 .../TemplateElasticSearchIndexStrategy.java | 74 +- .../TimeByDayElasticSearchIndexStrategy.java | 19 +- ...TimeByMonthElasticSearchIndexStrategy.java | 5 +- .../TimeByWeekElasticSearchIndexStrategy.java | 35 + .../search/service/AggregationService.java | 7 + .../search/service/ElasticSearchService.java | 248 +++ .../search/service/reactive/AggType.java | 171 ++ .../DefaultReactiveElasticsearchClient.java | 67 + .../ElasticSearchBufferProperties.java | 3 +- .../reactive/ReactiveAggregationService.java | 450 +++++ .../ReactiveElasticSearchService.java | 538 +++--- .../reactive/ReactiveElasticsearchClient.java | 42 + .../service/reactive/ScrollingFlux.java | 230 +++ .../ElasticSearchColumnModeDDLOperations.java | 1 - ...lasticSearchColumnModeQueryOperations.java | 25 +- ...ElasticSearchColumnModeSaveOperations.java | 0 .../ElasticSearchColumnModeStrategy.java | 3 + .../ElasticSearchRowModeDDLOperations.java | 12 +- .../ElasticSearchRowModeQueryOperations.java | 22 +- .../ElasticSearchRowModeSaveOperations.java | 66 + .../things/ElasticSearchRowModeStrategy.java | 3 + .../ElasticSearchTimeSeriesManager.java | 12 +- .../ElasticSearchTimeSeriesService.java | 56 +- .../search/trace/TraceInstrumentation.java | 138 ++ .../search/utils/ElasticSearchConverter.java | 15 +- .../search/utils/QueryParamTranslator.java | 295 +++ ...ot.autoconfigure.AutoConfiguration.imports | 1 + .../elasticsearch-component/pom.xml | 74 +- .../elasticsearch/common/logging/Loggers.java | 148 -- .../bucket/AggregationResponseHandle.java | 171 -- .../search/aggreation/bucket/Bucket.java | 70 - .../bucket/BucketAggregationsStructure.java | 70 - .../aggreation/bucket/BucketResponse.java | 21 - .../bucket/DateHistogramInterval.java | 38 - .../search/aggreation/bucket/Ranges.java | 19 - .../search/aggreation/bucket/Sort.java | 54 - .../aggreation/enums/AggregationType.java | 28 - .../search/aggreation/enums/BucketType.java | 199 -- .../search/aggreation/enums/MetricsType.java | 136 -- .../search/aggreation/enums/OrderType.java | 18 - .../metrics/MetricsAggregationStructure.java | 37 - .../aggreation/metrics/MetricsResponse.java | 33 - .../metrics/MetricsResponseSingleValue.java | 47 - .../ElasticSearchProperties.java | 51 - .../embedded/EmbeddedElasticSearch.java | 33 - .../EmbeddedElasticSearchProperties.java | 30 - .../elastic/search/enums/LinkTypeEnum.java | 69 - .../elastic/search/enums/TermTypeEnum.java | 114 -- .../index/ElasticSearchIndexProperties.java | 33 - .../search/index/IndexTemplateProvider.java | 12 - .../index/mapping/IndexMappingMetadata.java | 64 - .../index/mapping/SingleMappingMetadata.java | 23 - .../AbstractElasticSearchIndexStrategy.java | 257 --- .../search/parser/DefaultLinkTypeParser.java | 73 - .../search/parser/DefaultTermTypeParser.java | 39 - .../elastic/search/parser/LinkTypeParser.java | 15 - .../elastic/search/parser/TermTypeParser.java | 17 - .../elastic/search/parser/TermsHandler.java | 71 - .../search/service/ElasticSearchService.java | 99 - .../search/service/reactive/AggType.java | 101 -- .../DefaultReactiveElasticsearchClient.java | 1599 ----------------- .../service/reactive/RawActionResponse.java | 70 - .../reactive/ReactiveAggregationService.java | 356 ---- .../reactive/ReactiveElasticsearchClient.java | 31 - .../ElasticSearchRowModeSaveOperations.java | 33 - .../search/utils/QueryParamTranslator.java | 125 -- .../search/utils/ReactorActionListener.java | 62 - .../elastic/search/utils/TermCommonUtils.java | 36 - jetlinks-components/gateway-component/pom.xml | 2 +- .../gateway/DeviceGatewayHelper.java | 2 +- .../socket/WebSocketMessagingHandler.java | 4 +- .../EventBusDispatcherConfiguration.java | 17 + .../gateway/spring/ProxyMessageListener.java | 59 +- .../gateway/spring/SpringMessageBroker.java | 152 +- .../supports/DeviceGatewayProperties.java | 2 +- ...ot.autoconfigure.AutoConfiguration.imports | 3 +- jetlinks-components/io-component/pom.xml | 2 +- jetlinks-components/logging-component/pom.xml | 4 +- .../logging/access/AccessLoggerService.java | 40 + .../access/AccessLoggingTranslator.java | 12 +- .../logging/access/SerializableAccessLog.java | 43 +- .../access/TimeSeriesAccessLoggerService.java | 90 + .../configuration/LoggingConfiguration.java | 53 +- .../handler/AccessLoggerEventHandler.java | 33 +- .../event/handler/LoggerIndexProvider.java | 19 - .../handler/SystemLoggerEventHandler.java | 44 +- .../logback/ShortenedThrowableConverter.java | 82 + .../logback/SystemLoggingAppender.java | 113 +- .../logging/system/SerializableSystemLog.java | 7 + .../logging/system/SystemLoggerService.java | 40 + .../system/TimeSeriesSystemLoggerService.java | 87 + .../community/logging/utils/LoggingUtil.java | 30 + ...ot.autoconfigure.AutoConfiguration.imports | 1 + .../network-component/http-component/pom.xml | 2 +- .../network/http/server/HttpExchange.java | 2 +- .../network-component/mqtt-component/pom.xml | 2 +- .../network-component/network-core/pom.xml | 2 +- .../network/AbstractServerNetworkConfig.java | 2 +- ...anager.java => ClusterNetworkManager.java} | 151 +- .../network/NetworkConfiguration.java | 19 +- .../community/network/NetworkProvider.java | 3 + jetlinks-components/network-component/pom.xml | 2 +- .../network-component/tcp-component/pom.xml | 2 +- .../notify-component/notify-core/pom.xml | 2 +- .../notify/DefaultNotifierManager.java | 49 +- .../notify/StaticTemplateManager.java | 20 +- .../NotifierAutoConfiguration.java | 16 +- .../notify/template/VariableDefinition.java | 2 +- ...t.autoconfigure.AutoConfiguration.imports} | 3 +- .../notify-component/messages_en.properties | 27 + .../notify-component/messages_zh.properties | 26 + .../notify-component/notify-dingtalk/pom.xml | 2 +- .../corp/DingTalkMessageTemplate.java | 2 +- .../dingtalk/corp/DingTalkProperties.java | 2 +- .../notify-component/notify-email/pom.xml | 2 +- .../email/embedded/DefaultEmailNotifier.java | 46 +- .../DefaultEmailNotifierProvider.java | 30 +- .../embedded/DefaultEmailProperties.java | 1 - .../DefaultEmailTemplateProvider.java | 110 ++ .../notify/email/embedded/EmailTemplate.java | 16 +- .../notify-component/notify-sms/pom.xml | 2 +- .../notify/sms/aliyun/AliyunSmsTemplate.java | 2 +- .../notify-component/notify-voice/pom.xml | 2 +- .../voice/aliyun/AliyunVoiceTemplate.java | 2 +- .../notify-component/notify-webhook/pom.xml | 2 +- .../webhook/http/HttpWebHookProperties.java | 2 +- .../notify-component/notify-wechat/pom.xml | 2 +- .../wechat/corp/WechatCorpProperties.java | 2 +- .../wechat/corp/WechatMessageTemplate.java | 2 +- jetlinks-components/notify-component/pom.xml | 2 +- jetlinks-components/pom.xml | 4 +- .../protocol-component/pom.xml | 2 +- .../protocol/ProtocolSupportEntity.java | 2 +- .../configuration/LazyProtocolSupports.java | 29 +- .../ProtocolAutoConfiguration.java | 3 - .../relation-component/pom.xml | 2 +- .../relation/entity/RelatedEntity.java | 2 +- .../rule-engine-component/pom.xml | 20 +- .../cluster/ClusterSchedulerLoadBalancer.java | 66 + .../RuleEngineConfiguration.java | 150 +- .../RuleEngineLogIndexInitialize.java | 39 - .../configuration/RuleEngineProperties.java | 24 + .../configuration/SpringClusterWorker.java | 28 + .../engine/entity/TaskSnapshotEntity.java | 88 + .../RuleEngineLoggerIndexProvider.java | 21 - .../engine/event/handler/RuleLogHandler.java | 35 - .../executor/device/DeviceSelectorSpec.java | 4 +- .../rule/engine/io/EventBusRuleIOManager.java | 117 ++ .../rule/engine/log/RuleEngineLogService.java | 32 + .../log/TimeSeriesRuleEngineLogService.java | 113 ++ .../LocalTaskSnapshotRepository.java | 122 ++ ...ot.autoconfigure.AutoConfiguration.imports | 1 + jetlinks-components/script-component/pom.xml | 13 +- .../script/JavaScriptFactoryTest.java | 2 - .../tdengine-component/pom.xml | 2 +- .../community/tdengine/TDEngineUtils.java | 2 +- .../tdengine/TDengineProperties.java | 2 +- jetlinks-components/things-component/pom.xml | 8 +- .../AutoRegisterThingsRegistry.java | 21 +- .../things/data/PropertyAggregation.java | 4 +- .../things/data/TableSafeMetricBuilder.java | 68 + .../things/data/ThingsDataUtils.java | 67 + .../things/data/operations/DataSettings.java | 3 + .../operations/TableSafeMetricBuilder.java | 67 + .../things/utils/ThingsDatabaseUtils.java | 61 +- .../timescaledb-component/pom.xml | 46 + .../timescaledb/TimescaleDBDataWriter.java | 14 + .../timescaledb/TimescaleDBOperations.java | 11 + .../timescaledb/TimescaleDBProperties.java | 42 + .../timescaledb/TimescaleDBUtils.java | 94 + .../TimescaleDBConfiguration.java | 23 + .../TimescaleDBThingsDataConfiguration.java | 39 + .../TimescaleDBTimeSeriesConfiguration.java | 28 + .../impl/DefaultTimescaleDBDataWriter.java | 183 ++ .../impl/DefaultTimescaleDBOperations.java | 109 ++ .../metadata/CreateHypertable.java | 34 + .../metadata/CreateRetentionPolicy.java | 32 + .../timescaledb/metadata/JsonbValueCodec.java | 50 + .../TimescaleDBAlterTableSqlBuilder.java | 23 + .../TimescaleDBCreateTableSqlBuilder.java | 47 + .../metadata/TimescaleDBDialect.java | 10 + .../metadata/TimescaleDBDialectProvider.java | 53 + .../TimescaleDBColumnModeDDLOperations.java | 130 ++ .../TimescaleDBColumnModeQueryOperations.java | 192 ++ .../TimescaleDBColumnModeSaveOperations.java | 35 + .../thing/TimescaleDBColumnModeStrategy.java | 80 + .../TimescaleDBRowModeDDLOperations.java | 148 ++ .../TimescaleDBRowModeQueryOperations.java | 203 +++ .../TimescaleDBRowModeSaveOperations.java | 35 + .../thing/TimescaleDBRowModeStrategy.java | 72 + .../TimescaleDBThingsDataProperties.java | 23 + .../TimescaleDBTimeSeriesManager.java | 127 ++ .../TimescaleDBTimeSeriesProperties.java | 51 + .../TimescaleDBTimeSeriesService.java | 191 ++ ...ork.web.crud.configuration.DialectProvider | 1 + ...ot.autoconfigure.AutoConfiguration.imports | 3 + .../timeseries-component/pom.xml | 2 +- .../timeseries/query/Aggregation.java | 23 +- .../timeseries/query/AggregationColumn.java | 4 +- .../timeseries/utils/TimeSeriesUtils.java | 109 ++ .../authentication-manager/pom.xml | 2 +- ...horizationPermissionInitializeService.java | 106 ++ .../AuthorizationProperties.java | 46 + .../CustomAuthenticationConfiguration.java | 10 +- .../community/auth/entity/MenuBindEntity.java | 2 +- .../auth/entity/OrganizationEntity.java | 2 +- .../auth/entity/ThirdPartyUserBindEntity.java | 2 +- .../auth/entity/UserDetailEntity.java | 6 +- .../auth/initialize/UserAutoInitialize.java | 69 + .../UserAutoInitializeProperties.java | 20 + .../community/auth/service/RoleService.java | 2 +- .../request/SaveUserDetailRequest.java | 2 +- .../auth/service/request/SaveUserRequest.java | 2 +- .../request/AuthorizationSettingDetail.java | 2 +- jetlinks-manager/device-manager/pom.xml | 4 +- .../device/entity/DeviceCategoryEntity.java | 4 +- .../device/entity/DeviceInstanceEntity.java | 4 +- .../entity/DeviceMetadataMappingEntity.java | 4 +- .../device/entity/DeviceProductEntity.java | 4 +- .../device/entity/DeviceTagEntity.java | 4 +- .../data/DeviceDataManagerSupport.java | 106 -- .../service/data/DeviceDataService.java | 4 +- .../service/data/DeviceLatestDataService.java | 4 +- .../device/web/excel/DeviceExcelInfo.java | 2 +- .../PropertyMetadataExcelImportInfo.java | 4 +- .../web/excel/PropertyMetadataExcelInfo.java | 2 +- .../TransparentMessageCodecRequest.java | 2 +- .../TransparentMessageDecodeRequest.java | 2 +- jetlinks-manager/logging-manager/pom.xml | 2 +- .../controller/AccessLoggerController.java | 20 +- .../controller/SystemLoggerController.java | 6 +- .../logging/service/AccessLoggerService.java | 26 - .../logging/service/SystemLoggerService.java | 25 - jetlinks-manager/network-manager/pom.xml | 2 +- .../TcpServerDebugSubscriptionProvider.java | 2 +- .../manager/entity/CertificateEntity.java | 2 +- .../manager/entity/DeviceGatewayEntity.java | 2 +- .../manager/entity/NetworkConfigEntity.java | 2 +- jetlinks-manager/notify-manager/pom.xml | 4 +- ...asticSearchNotifyHistoryConfiguration.java | 24 - .../configuration/NotifyConfiguration.java | 13 +- .../manager/entity/NotifyConfigEntity.java | 4 +- .../entity/NotifySubscriberChannelEntity.java | 2 +- .../NotifySubscriberProviderEntity.java | 2 +- .../manager/entity/NotifyTemplateEntity.java | 4 +- .../service/DefaultTemplateManager.java | 36 +- .../ElasticSearchNotifyHistoryRepository.java | 61 - .../InDatabaseNotifyHistoryRepository.java | 32 - .../TimeSeriesNotifyHistoryRepository.java | 78 + .../notifiers/NotifierChannelProvider.java | 2 +- .../manager/web/NotifierController.java | 2 +- jetlinks-manager/pom.xml | 2 +- jetlinks-manager/rule-engine-manager/pom.xml | 4 +- .../rule/engine/alarm/AlarmHandleInfo.java | 4 +- .../alarm/AlarmTaskExecutorProvider.java | 2 +- .../RuleEngineManagerConfiguration.java | 17 +- .../rule/engine/entity/AlarmConfigDetail.java | 2 +- .../engine/entity/AlarmRuleBindEntity.java | 2 +- .../engine/entity/RuleInstanceEntity.java | 2 +- .../rule/engine/entity/SceneEntity.java | 4 +- .../rule/engine/enums/SqlRuleType.java | 65 - .../community/rule/engine/model/Action.java | 28 - .../rule/engine/model/SqlRuleModelParser.java | 118 -- .../community/rule/engine/ql/SqlRule.java | 43 - .../rule/engine/scene/DeviceOperation.java | 2 +- .../rule/engine/scene/SceneAction.java | 2 +- .../rule/engine/scene/SceneRule.java | 4 +- .../community/rule/engine/scene/Trigger.java | 2 +- .../scene/internal/actions/NotifyAction.java | 2 +- .../internal/triggers/DeviceTrigger.java | 4 +- .../ElasticSearchAlarmHistoryService.java | 109 -- .../service/LocalRuleInstanceRepository.java | 18 +- .../engine/service/RuleInstanceService.java | 9 +- .../TimeSeriesAlarmHistoryService.java | 129 ++ .../messages_zh.properties | 2 - jetlinks-standalone/Dockerfile | 24 +- jetlinks-standalone/docker-entrypoint.sh | 22 +- jetlinks-standalone/pom.xml | 58 +- .../standalone/JetLinksApplication.java | 26 - .../main/resources/application-embedded.yml | 44 - .../src/main/resources/application.yml | 463 ++--- .../src/main/resources/banner.txt | 4 +- .../src/main/resources/hsweb-starter.js | 53 - .../src/main/resources/logback-spring.xml | 5 + pom.xml | 210 ++- 382 files changed, 12212 insertions(+), 7226 deletions(-) create mode 100644 .run/JetLinksApplication.run.xml delete mode 100755 jetlinks-components/common-component/src/main/java/org/jetlinks/community/reactorql/EnableReactorQL.java delete mode 100755 jetlinks-components/common-component/src/main/java/org/jetlinks/community/reactorql/ReactorQL.java delete mode 100755 jetlinks-components/common-component/src/main/java/org/jetlinks/community/reactorql/ReactorQLBeanDefinitionRegistrar.java delete mode 100755 jetlinks-components/common-component/src/main/java/org/jetlinks/community/reactorql/ReactorQLFactoryBean.java delete mode 100755 jetlinks-components/common-component/src/main/java/org/jetlinks/community/reactorql/ReactorQLOperation.java create mode 100644 jetlinks-components/configure-component/src/main/java/org/jetlinks/community/configure/compatible/PathCompatibleFilter.java create mode 100644 jetlinks-components/datasource-component/pom.xml create mode 100644 jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/AbstractDataSource.java create mode 100644 jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DataSource.java create mode 100644 jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DataSourceConfig.java create mode 100644 jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DataSourceConfigManager.java create mode 100644 jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DataSourceConstants.java create mode 100644 jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DataSourceManager.java create mode 100644 jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DataSourceProvider.java create mode 100644 jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DataSourceState.java create mode 100644 jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DataSourceType.java create mode 100644 jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DefaultDataSourceManager.java create mode 100644 jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/command/CommandHandlerProvider.java create mode 100644 jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/command/DataSourceCommandConfig.java create mode 100644 jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/command/DataSourceCommandSupportManager.java create mode 100644 jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/configuration/DataSourceHandlerProviderConfiguration.java create mode 100644 jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/configuration/DataSourceHandlerProviderRegister.java create mode 100644 jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/configuration/DataSourceManagerConfiguration.java create mode 100644 jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/exception/DataSourceNotExistException.java create mode 100644 jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/DefaultRDBDataSource.java create mode 100644 jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/RDBDataSource.java create mode 100644 jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/RDBDataSourceProperties.java create mode 100644 jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/RDBDataSourceProvider.java create mode 100644 jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/RDBDataSourceType.java create mode 100644 jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/RDBJdbcReactiveSqlExecutor.java create mode 100644 jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/RDBJdbcSyncSqlExecutor.java create mode 100644 jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/Column.java create mode 100644 jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/Count.java create mode 100644 jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/CreateOrAlterTable.java create mode 100644 jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/DropColumn.java create mode 100644 jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/ExecuteSql.java create mode 100644 jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/GetTable.java create mode 100644 jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/GetTables.java create mode 100644 jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/QueryList.java create mode 100644 jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/QueryPager.java create mode 100644 jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/RDBCommand.java create mode 100644 jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/RDBRequestListCommand.java create mode 100644 jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/RDBRequestPagerCommand.java create mode 100644 jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/Refresh.java create mode 100644 jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/Table.java create mode 100644 jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/Upsert.java create mode 100644 jetlinks-components/datasource-component/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports create mode 100644 jetlinks-components/datasource-component/src/main/resources/i18n/datasource/messages_en.properties create mode 100644 jetlinks-components/datasource-component/src/main/resources/i18n/datasource/messages_zh.properties create mode 100755 jetlinks-components/elasticsearch-component/.gitignore create mode 100644 jetlinks-components/elasticsearch-component/elasticsearch-7x/pom.xml create mode 100644 jetlinks-components/elasticsearch-component/elasticsearch-7x/src/main/java/org/jetlinks/community/elastic/search/ElasticSearch7xSupport.java create mode 100755 jetlinks-components/elasticsearch-component/elasticsearch-7x/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticSearch7xTermType.java create mode 100644 jetlinks-components/elasticsearch-component/elasticsearch-7x/src/main/resources/META-INF/services/org.jetlinks.community.elastic.search.ElasticSearchSupport create mode 100644 jetlinks-components/elasticsearch-component/elasticsearch-8x/pom.xml create mode 100644 jetlinks-components/elasticsearch-component/elasticsearch-8x/src/main/java/org/jetlinks/community/elastic/search/ElasticSearch8xSupport.java create mode 100755 jetlinks-components/elasticsearch-component/elasticsearch-8x/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticSearch8xTermType.java create mode 100644 jetlinks-components/elasticsearch-component/elasticsearch-8x/src/main/resources/META-INF/services/org.jetlinks.community.elastic.search.ElasticSearchSupport create mode 100644 jetlinks-components/elasticsearch-component/elasticsearch-core/pom.xml create mode 100644 jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/ElasticSearchSupport.java create mode 100755 jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/configuration/ElasticSearchClientConfiguration.java rename jetlinks-components/elasticsearch-component/{ => elasticsearch-core}/src/main/java/org/jetlinks/community/elastic/search/configuration/ElasticSearchConfiguration.java (53%) create mode 100644 jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/configuration/ElasticSearchProperties.java rename jetlinks-components/elasticsearch-component/{ => elasticsearch-core}/src/main/java/org/jetlinks/community/elastic/search/configuration/ElasticSearchThingDataConfiguration.java (90%) rename jetlinks-components/elasticsearch-component/{ => elasticsearch-core}/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticDateFormat.java (84%) mode change 100644 => 100755 rename jetlinks-components/elasticsearch-component/{ => elasticsearch-core}/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticPropertyType.java (74%) mode change 100644 => 100755 create mode 100644 jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticSearchTermType.java create mode 100644 jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticSearchTermTypes.java rename jetlinks-components/elasticsearch-component/{ => elasticsearch-core}/src/main/java/org/jetlinks/community/elastic/search/index/DefaultElasticSearchIndexManager.java (79%) rename jetlinks-components/elasticsearch-component/{ => elasticsearch-core}/src/main/java/org/jetlinks/community/elastic/search/index/DefaultElasticSearchIndexMetadata.java (82%) mode change 100644 => 100755 rename jetlinks-components/elasticsearch-component/{ => elasticsearch-core}/src/main/java/org/jetlinks/community/elastic/search/index/ElasticIndex.java (100%) mode change 100644 => 100755 create mode 100755 jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexCustomizer.java rename jetlinks-components/elasticsearch-component/{ => elasticsearch-core}/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexManager.java (85%) mode change 100644 => 100755 rename jetlinks-components/elasticsearch-component/{ => elasticsearch-core}/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexMetadata.java (100%) mode change 100644 => 100755 create mode 100755 jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexProperties.java rename jetlinks-components/elasticsearch-component/{ => elasticsearch-core}/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexStrategy.java (89%) mode change 100644 => 100755 create mode 100755 jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/strategies/AbstractElasticSearchIndexStrategy.java create mode 100644 jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/strategies/AffixesElasticSearchIndexStrategy.java rename jetlinks-components/elasticsearch-component/{ => elasticsearch-core}/src/main/java/org/jetlinks/community/elastic/search/index/strategies/DirectElasticSearchIndexStrategy.java (100%) rename jetlinks-components/elasticsearch-component/{ => elasticsearch-core}/src/main/java/org/jetlinks/community/elastic/search/index/strategies/TemplateElasticSearchIndexStrategy.java (52%) rename jetlinks-components/elasticsearch-component/{ => elasticsearch-core}/src/main/java/org/jetlinks/community/elastic/search/index/strategies/TimeByDayElasticSearchIndexStrategy.java (60%) mode change 100644 => 100755 rename jetlinks-components/elasticsearch-component/{ => elasticsearch-core}/src/main/java/org/jetlinks/community/elastic/search/index/strategies/TimeByMonthElasticSearchIndexStrategy.java (85%) create mode 100755 jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/strategies/TimeByWeekElasticSearchIndexStrategy.java rename jetlinks-components/elasticsearch-component/{ => elasticsearch-core}/src/main/java/org/jetlinks/community/elastic/search/service/AggregationService.java (77%) mode change 100644 => 100755 create mode 100755 jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/service/ElasticSearchService.java create mode 100755 jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/service/reactive/AggType.java create mode 100755 jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/service/reactive/DefaultReactiveElasticsearchClient.java rename jetlinks-components/elasticsearch-component/{ => elasticsearch-core}/src/main/java/org/jetlinks/community/elastic/search/service/reactive/ElasticSearchBufferProperties.java (91%) create mode 100755 jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/service/reactive/ReactiveAggregationService.java rename jetlinks-components/elasticsearch-component/{ => elasticsearch-core}/src/main/java/org/jetlinks/community/elastic/search/service/reactive/ReactiveElasticSearchService.java (53%) create mode 100755 jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/service/reactive/ReactiveElasticsearchClient.java create mode 100644 jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/service/reactive/ScrollingFlux.java rename jetlinks-components/elasticsearch-component/{ => elasticsearch-core}/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchColumnModeDDLOperations.java (97%) rename jetlinks-components/elasticsearch-component/{ => elasticsearch-core}/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchColumnModeQueryOperations.java (87%) rename jetlinks-components/elasticsearch-component/{ => elasticsearch-core}/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchColumnModeSaveOperations.java (100%) rename jetlinks-components/elasticsearch-component/{ => elasticsearch-core}/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchColumnModeStrategy.java (94%) rename jetlinks-components/elasticsearch-component/{ => elasticsearch-core}/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchRowModeDDLOperations.java (94%) rename jetlinks-components/elasticsearch-component/{ => elasticsearch-core}/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchRowModeQueryOperations.java (92%) create mode 100644 jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchRowModeSaveOperations.java rename jetlinks-components/elasticsearch-component/{ => elasticsearch-core}/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchRowModeStrategy.java (94%) rename jetlinks-components/elasticsearch-component/{ => elasticsearch-core}/src/main/java/org/jetlinks/community/elastic/search/timeseries/ElasticSearchTimeSeriesManager.java (89%) mode change 100644 => 100755 rename jetlinks-components/elasticsearch-component/{ => elasticsearch-core}/src/main/java/org/jetlinks/community/elastic/search/timeseries/ElasticSearchTimeSeriesService.java (62%) mode change 100644 => 100755 create mode 100644 jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/trace/TraceInstrumentation.java rename jetlinks-components/elasticsearch-component/{ => elasticsearch-core}/src/main/java/org/jetlinks/community/elastic/search/utils/ElasticSearchConverter.java (85%) mode change 100644 => 100755 create mode 100755 jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/utils/QueryParamTranslator.java rename jetlinks-components/elasticsearch-component/{ => elasticsearch-core}/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports (56%) mode change 100644 => 100755 jetlinks-components/elasticsearch-component/pom.xml delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/elasticsearch/common/logging/Loggers.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/bucket/AggregationResponseHandle.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/bucket/Bucket.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/bucket/BucketAggregationsStructure.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/bucket/BucketResponse.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/bucket/DateHistogramInterval.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/bucket/Ranges.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/bucket/Sort.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/enums/AggregationType.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/enums/BucketType.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/enums/MetricsType.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/enums/OrderType.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/metrics/MetricsAggregationStructure.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/metrics/MetricsResponse.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/metrics/MetricsResponseSingleValue.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/configuration/ElasticSearchProperties.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/embedded/EmbeddedElasticSearch.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/embedded/EmbeddedElasticSearchProperties.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/enums/LinkTypeEnum.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/enums/TermTypeEnum.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexProperties.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/IndexTemplateProvider.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/mapping/IndexMappingMetadata.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/mapping/SingleMappingMetadata.java delete mode 100755 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/strategies/AbstractElasticSearchIndexStrategy.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/parser/DefaultLinkTypeParser.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/parser/DefaultTermTypeParser.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/parser/LinkTypeParser.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/parser/TermTypeParser.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/parser/TermsHandler.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/ElasticSearchService.java delete mode 100755 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/reactive/AggType.java delete mode 100755 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/reactive/DefaultReactiveElasticsearchClient.java delete mode 100755 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/reactive/RawActionResponse.java delete mode 100755 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/reactive/ReactiveAggregationService.java delete mode 100755 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/reactive/ReactiveElasticsearchClient.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchRowModeSaveOperations.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/utils/QueryParamTranslator.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/utils/ReactorActionListener.java delete mode 100644 jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/utils/TermCommonUtils.java create mode 100644 jetlinks-components/gateway-component/src/main/java/org/jetlinks/community/gateway/spring/EventBusDispatcherConfiguration.java create mode 100755 jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/access/AccessLoggerService.java mode change 100644 => 100755 jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/access/AccessLoggingTranslator.java mode change 100644 => 100755 jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/access/SerializableAccessLog.java create mode 100644 jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/access/TimeSeriesAccessLoggerService.java mode change 100644 => 100755 jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/event/handler/AccessLoggerEventHandler.java delete mode 100644 jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/event/handler/LoggerIndexProvider.java create mode 100644 jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/logback/ShortenedThrowableConverter.java mode change 100644 => 100755 jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/system/SerializableSystemLog.java create mode 100755 jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/system/SystemLoggerService.java create mode 100644 jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/system/TimeSeriesSystemLoggerService.java create mode 100644 jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/utils/LoggingUtil.java create mode 100644 jetlinks-components/logging-component/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports rename jetlinks-components/network-component/network-core/src/main/java/org/jetlinks/community/network/{DefaultNetworkManager.java => ClusterNetworkManager.java} (51%) mode change 100644 => 100755 rename jetlinks-components/notify-component/notify-core/src/main/resources/META-INF/{spring.factories => spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports} (60%) mode change 100755 => 100644 create mode 100644 jetlinks-components/notify-component/notify-core/src/main/resources/i18n/notify-component/messages_en.properties create mode 100644 jetlinks-components/notify-component/notify-core/src/main/resources/i18n/notify-component/messages_zh.properties create mode 100755 jetlinks-components/notify-component/notify-email/src/main/java/org/jetlinks/community/notify/email/embedded/DefaultEmailTemplateProvider.java create mode 100755 jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/cluster/ClusterSchedulerLoadBalancer.java delete mode 100644 jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/configuration/RuleEngineLogIndexInitialize.java create mode 100755 jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/configuration/RuleEngineProperties.java create mode 100644 jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/configuration/SpringClusterWorker.java create mode 100755 jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/entity/TaskSnapshotEntity.java delete mode 100644 jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/event/handler/RuleEngineLoggerIndexProvider.java delete mode 100644 jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/event/handler/RuleLogHandler.java create mode 100644 jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/io/EventBusRuleIOManager.java create mode 100755 jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/log/RuleEngineLogService.java create mode 100755 jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/log/TimeSeriesRuleEngineLogService.java create mode 100755 jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/repository/LocalTaskSnapshotRepository.java create mode 100644 jetlinks-components/rule-engine-component/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports create mode 100644 jetlinks-components/things-component/src/main/java/org/jetlinks/community/things/data/TableSafeMetricBuilder.java create mode 100644 jetlinks-components/things-component/src/main/java/org/jetlinks/community/things/data/ThingsDataUtils.java create mode 100644 jetlinks-components/things-component/src/main/java/org/jetlinks/community/things/data/operations/TableSafeMetricBuilder.java create mode 100644 jetlinks-components/timescaledb-component/pom.xml create mode 100644 jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/TimescaleDBDataWriter.java create mode 100644 jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/TimescaleDBOperations.java create mode 100644 jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/TimescaleDBProperties.java create mode 100644 jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/TimescaleDBUtils.java create mode 100644 jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/configuration/TimescaleDBConfiguration.java create mode 100644 jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/configuration/TimescaleDBThingsDataConfiguration.java create mode 100644 jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/configuration/TimescaleDBTimeSeriesConfiguration.java create mode 100644 jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/impl/DefaultTimescaleDBDataWriter.java create mode 100644 jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/impl/DefaultTimescaleDBOperations.java create mode 100644 jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/metadata/CreateHypertable.java create mode 100644 jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/metadata/CreateRetentionPolicy.java create mode 100644 jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/metadata/JsonbValueCodec.java create mode 100644 jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/metadata/TimescaleDBAlterTableSqlBuilder.java create mode 100644 jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/metadata/TimescaleDBCreateTableSqlBuilder.java create mode 100644 jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/metadata/TimescaleDBDialect.java create mode 100644 jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/metadata/TimescaleDBDialectProvider.java create mode 100644 jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/thing/TimescaleDBColumnModeDDLOperations.java create mode 100644 jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/thing/TimescaleDBColumnModeQueryOperations.java create mode 100644 jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/thing/TimescaleDBColumnModeSaveOperations.java create mode 100644 jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/thing/TimescaleDBColumnModeStrategy.java create mode 100644 jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/thing/TimescaleDBRowModeDDLOperations.java create mode 100644 jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/thing/TimescaleDBRowModeQueryOperations.java create mode 100644 jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/thing/TimescaleDBRowModeSaveOperations.java create mode 100644 jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/thing/TimescaleDBRowModeStrategy.java create mode 100644 jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/thing/TimescaleDBThingsDataProperties.java create mode 100644 jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/timeseries/TimescaleDBTimeSeriesManager.java create mode 100644 jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/timeseries/TimescaleDBTimeSeriesProperties.java create mode 100644 jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/timeseries/TimescaleDBTimeSeriesService.java create mode 100644 jetlinks-components/timescaledb-component/src/main/resources/META-INF/services/org.hswebframework.web.crud.configuration.DialectProvider create mode 100644 jetlinks-components/timescaledb-component/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports create mode 100644 jetlinks-components/timeseries-component/src/main/java/org/jetlinks/community/timeseries/utils/TimeSeriesUtils.java create mode 100644 jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/configuration/AuthorizationPermissionInitializeService.java create mode 100644 jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/configuration/AuthorizationProperties.java create mode 100644 jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/initialize/UserAutoInitialize.java create mode 100644 jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/initialize/UserAutoInitializeProperties.java delete mode 100644 jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/service/data/DeviceDataManagerSupport.java delete mode 100644 jetlinks-manager/logging-manager/src/main/java/org/jetlinks/community/logging/service/AccessLoggerService.java delete mode 100644 jetlinks-manager/logging-manager/src/main/java/org/jetlinks/community/logging/service/SystemLoggerService.java delete mode 100644 jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/configuration/ElasticSearchNotifyHistoryConfiguration.java delete mode 100644 jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/service/ElasticSearchNotifyHistoryRepository.java delete mode 100644 jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/service/InDatabaseNotifyHistoryRepository.java create mode 100644 jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/service/TimeSeriesNotifyHistoryRepository.java delete mode 100644 jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/enums/SqlRuleType.java delete mode 100644 jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/model/Action.java delete mode 100644 jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/model/SqlRuleModelParser.java delete mode 100644 jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/ql/SqlRule.java delete mode 100644 jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/service/ElasticSearchAlarmHistoryService.java create mode 100644 jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/service/TimeSeriesAlarmHistoryService.java delete mode 100644 jetlinks-standalone/src/main/resources/application-embedded.yml delete mode 100644 jetlinks-standalone/src/main/resources/hsweb-starter.js diff --git a/.gitignore b/.gitignore index a93943bc..2280b620 100644 --- a/.gitignore +++ b/.gitignore @@ -29,4 +29,5 @@ docker/data !demo-protocol-1.0.jar application-local.yml dev/ -.DS_Store \ No newline at end of file +.DS_Store +.java-version \ No newline at end of file diff --git a/.run/JetLinksApplication.run.xml b/.run/JetLinksApplication.run.xml new file mode 100644 index 00000000..c6d1c856 --- /dev/null +++ b/.run/JetLinksApplication.run.xml @@ -0,0 +1,18 @@ + + + + \ No newline at end of file diff --git a/README.md b/README.md index d3c8113c..8db0f9e2 100644 --- a/README.md +++ b/README.md @@ -14,10 +14,10 @@ [![QQ②群324606263](https://img.shields.io/badge/QQ②群-324606263-brightgreen)](https://qm.qq.com/cgi-bin/qm/qr?k=IMas2cH-TNsYxUcY8lRbsXqPnA2sGHYQ&jump_from=webapi) [![QQ①群2021514](https://img.shields.io/badge/QQ①群-2021514-brightgreen)](https://qm.qq.com/cgi-bin/qm/qr?k=LGf0OPQqvLGdJIZST3VTcypdVWhdfAOG&jump_from=webapi) -JetLinks 基于Java8,Spring Boot 2.x,WebFlux,Netty,Vert.x,Reactor等开发, +JetLinks 基于Java 17,Spring Boot 3.x,WebFlux,Netty,Vert.x,Reactor等开发, 是一个开箱即用,可二次开发的企业级物联网基础平台。平台实现了物联网相关的众多基础功能, 能帮助你快速建立物联网相关业务系统。 - + ## 核心特性 @@ -36,7 +36,7 @@ TCP/UDP/MQTT/HTTP、TLS/DTLS、不同厂商、不同设备、不同报文、统 ## 技术栈 -1. [Spring Boot 2.7.x](https://spring.io/projects/spring-boot) +1. [Spring Boot 3.4.x](https://spring.io/projects/spring-boot) 2. [Spring WebFlux](https://spring.io/) 响应式Web支持 3. [R2DBC](https://r2dbc.io/) 响应式关系型数据库驱动 4. [Project Reactor](https://projectreactor.io/) 响应式编程框架 diff --git a/docker/run-all/docker-compose.yml b/docker/run-all/docker-compose.yml index bdcfbe80..f975080a 100644 --- a/docker/run-all/docker-compose.yml +++ b/docker/run-all/docker-compose.yml @@ -48,7 +48,7 @@ services: POSTGRES_DB: jetlinks TZ: Asia/Shanghai ui: - image: registry.cn-shenzhen.aliyuncs.com/jetlinks/jetlinks-ui-vue:2.3.0-SNAPSHOT + image: registry.cn-shenzhen.aliyuncs.com/jetlinks/jetlinks-ui-vue:2.10.0-SNAPSHOT container_name: jetlinks-ce-ui ports: - 9000:80 @@ -59,7 +59,7 @@ services: links: - jetlinks:jetlinks jetlinks: - image: registry.cn-shenzhen.aliyuncs.com/jetlinks/jetlinks-community:2.3.0-SNAPSHOT + image: registry.cn-shenzhen.aliyuncs.com/jetlinks/jetlinks-community:2.10.0-SNAPSHOT container_name: jetlinks-ce ports: diff --git a/jetlinks-components/common-component/pom.xml b/jetlinks-components/common-component/pom.xml index 1244c294..5223e7aa 100644 --- a/jetlinks-components/common-component/pom.xml +++ b/jetlinks-components/common-component/pom.xml @@ -5,7 +5,7 @@ jetlinks-components org.jetlinks.community - 2.3.0-SNAPSHOT + 2.10.0-SNAPSHOT ../pom.xml 4.0.0 @@ -80,9 +80,12 @@ com.cronutils cron-utils - 9.2.0 compile + + org.glassfish + jakarta.el + org.glassfish javax.el @@ -95,5 +98,11 @@ spring-data-redis true + + + org.glassfish.expressly + expressly + + diff --git a/jetlinks-components/common-component/src/main/java/org/jetlinks/community/PropertyConstants.java b/jetlinks-components/common-component/src/main/java/org/jetlinks/community/PropertyConstants.java index 804166c4..7280c890 100644 --- a/jetlinks-components/common-component/src/main/java/org/jetlinks/community/PropertyConstants.java +++ b/jetlinks-components/common-component/src/main/java/org/jetlinks/community/PropertyConstants.java @@ -38,13 +38,6 @@ public interface PropertyConstants { */ Key>> relations = Key.of("relations"); - /** - * 租户ID - * - * @see org.jetlinks.pro.tenant.TenantMember - */ - Key> tenantId = Key.of("tenantId"); - //分组ID Key> groupId = Key.of("groupId"); @@ -71,7 +64,7 @@ public interface PropertyConstants { /** * 设备接入方式 * - * @see org.jetlinks.pro.gateway.supports.DeviceGatewayProvider#getId + * @see org.jetlinks.community.gateway.supports.DeviceGatewayProvider#getId */ Key accessProvider = Key.of("accessProvider"); diff --git a/jetlinks-components/common-component/src/main/java/org/jetlinks/community/PropertyMetric.java b/jetlinks-components/common-component/src/main/java/org/jetlinks/community/PropertyMetric.java index 09bedae4..00a3ec75 100644 --- a/jetlinks-components/common-component/src/main/java/org/jetlinks/community/PropertyMetric.java +++ b/jetlinks-components/common-component/src/main/java/org/jetlinks/community/PropertyMetric.java @@ -6,7 +6,7 @@ import org.hswebframework.web.bean.FastBeanCopier; import org.jetlinks.community.utils.ConverterUtils; import org.springframework.util.StringUtils; -import javax.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotBlank; import java.util.Map; import java.util.function.Function; diff --git a/jetlinks-components/common-component/src/main/java/org/jetlinks/community/TimerSpec.java b/jetlinks-components/common-component/src/main/java/org/jetlinks/community/TimerSpec.java index 285ef9d3..a7026e3f 100644 --- a/jetlinks-components/common-component/src/main/java/org/jetlinks/community/TimerSpec.java +++ b/jetlinks-components/common-component/src/main/java/org/jetlinks/community/TimerSpec.java @@ -28,8 +28,8 @@ import reactor.core.scheduler.Scheduler; import reactor.core.scheduler.Schedulers; import javax.annotation.Nonnull; -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.NotNull; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; import java.io.Serializable; import java.time.*; import java.time.temporal.ChronoUnit; diff --git a/jetlinks-components/common-component/src/main/java/org/jetlinks/community/configuration/CommonConfiguration.java b/jetlinks-components/common-component/src/main/java/org/jetlinks/community/configuration/CommonConfiguration.java index b648f099..2af82b8e 100644 --- a/jetlinks-components/common-component/src/main/java/org/jetlinks/community/configuration/CommonConfiguration.java +++ b/jetlinks-components/common-component/src/main/java/org/jetlinks/community/configuration/CommonConfiguration.java @@ -13,6 +13,8 @@ import org.hswebframework.web.dict.EnumDict; import org.hswebframework.web.dict.defaults.DefaultItemDefine; import org.jetlinks.community.Interval; import org.jetlinks.community.JvmErrorException; +import org.jetlinks.community.command.CommandSupportManagerProvider; +import org.jetlinks.community.command.CommandSupportManagerProviders; import org.jetlinks.community.command.register.CommandServiceEndpointRegister; import org.jetlinks.community.config.ConfigManager; import org.jetlinks.community.config.ConfigScopeCustomizer; @@ -20,8 +22,11 @@ import org.jetlinks.community.config.ConfigScopeProperties; import org.jetlinks.community.config.SimpleConfigManager; import org.jetlinks.community.config.entity.ConfigEntity; import org.jetlinks.community.dictionary.DictionaryJsonDeserializer; +import org.jetlinks.community.form.type.FieldTypeProvider; import org.jetlinks.community.reactorql.aggregation.InternalAggregationSupports; import org.jetlinks.community.reactorql.function.InternalFunctionSupport; +import org.jetlinks.community.reactorql.term.TermTypeSupport; +import org.jetlinks.community.reactorql.term.TermTypes; import org.jetlinks.community.reference.DataReferenceManager; import org.jetlinks.community.reference.DataReferenceProvider; import org.jetlinks.community.reference.DefaultDataReferenceManager; @@ -45,6 +50,7 @@ import org.springframework.boot.autoconfigure.AutoConfiguration; import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; import org.springframework.boot.autoconfigure.jackson.Jackson2ObjectMapperBuilderCustomizer; import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.ApplicationContextAware; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.data.redis.core.ReactiveRedisOperations; @@ -185,15 +191,20 @@ public class CommonConfiguration { } @Bean - public BeanPostProcessor globalReactorQlFeatureRegister() { - return new BeanPostProcessor() { - @Override - public Object postProcessAfterInitialization(@Nonnull Object bean, @Nonnull String beanName) throws BeansException { - if (bean instanceof Feature) { - DefaultReactorQLMetadata.addGlobal(((Feature) bean)); - } - return bean; - } + public ApplicationContextAware staticBeanRegister() { + + return ctx -> { + ctx.getBeanProvider(Feature.class) + .forEach(DefaultReactorQLMetadata::addGlobal); + + ctx.getBeanProvider(CommandSupportManagerProvider.class) + .forEach(CommandSupportManagerProviders::register); + + ctx.getBeanProvider(TermTypeSupport.class) + .forEach(TermTypes::register); + + ctx.getBeanProvider(FieldTypeProvider.class) + .forEach(provider -> FieldTypeProvider.supports.register(provider.getProvider(), provider)); }; } diff --git a/jetlinks-components/common-component/src/main/java/org/jetlinks/community/dictionary/DictionaryConfiguration.java b/jetlinks-components/common-component/src/main/java/org/jetlinks/community/dictionary/DictionaryConfiguration.java index 8a17ea37..d7a6ada0 100644 --- a/jetlinks-components/common-component/src/main/java/org/jetlinks/community/dictionary/DictionaryConfiguration.java +++ b/jetlinks-components/common-component/src/main/java/org/jetlinks/community/dictionary/DictionaryConfiguration.java @@ -1,5 +1,8 @@ package org.jetlinks.community.dictionary; +import org.hswebframework.web.crud.events.EntityEventListenerCustomizer; +import org.hswebframework.web.dictionary.entity.DictionaryEntity; +import org.hswebframework.web.dictionary.entity.DictionaryItemEntity; import org.hswebframework.web.dictionary.service.DefaultDictionaryItemService; import org.hswebframework.web.dictionary.service.DefaultDictionaryService; import org.springframework.beans.factory.ObjectProvider; @@ -9,15 +12,22 @@ import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -@Configuration +@AutoConfiguration public class DictionaryConfiguration { - @Configuration + @AutoConfiguration @ConditionalOnClass(DefaultDictionaryItemService.class) //@ConditionalOnBean(DefaultDictionaryItemService.class) public static class DictionaryManagerConfiguration { + @Bean + public EntityEventListenerCustomizer dictionaryEntityEventListenerCustomizer() { + return configure -> { + configure.enable(DictionaryItemEntity.class); + configure.enable(DictionaryEntity.class); + }; + } @Bean public DictionaryEventHandler dictionaryEventHandler(DefaultDictionaryItemService service) { @@ -44,5 +54,6 @@ public class DictionaryConfiguration { DefaultDictionaryItemService itemService) { return new DictionaryInitManager(initInfo, defaultDictionaryService, itemService); } + } } diff --git a/jetlinks-components/common-component/src/main/java/org/jetlinks/community/dictionary/DictionaryEventHandler.java b/jetlinks-components/common-component/src/main/java/org/jetlinks/community/dictionary/DictionaryEventHandler.java index 5a3670a3..1bca258c 100644 --- a/jetlinks-components/common-component/src/main/java/org/jetlinks/community/dictionary/DictionaryEventHandler.java +++ b/jetlinks-components/common-component/src/main/java/org/jetlinks/community/dictionary/DictionaryEventHandler.java @@ -7,16 +7,21 @@ import org.hswebframework.web.crud.events.*; import org.hswebframework.web.dictionary.entity.DictionaryEntity; import org.hswebframework.web.dictionary.entity.DictionaryItemEntity; import org.hswebframework.web.dictionary.service.DefaultDictionaryItemService; +import org.hswebframework.web.dictionary.service.DefaultDictionaryService; import org.hswebframework.web.exception.BusinessException; import org.springframework.context.event.EventListener; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; +import java.util.Collections; +import java.util.List; + /** * @author bestfeng */ @AllArgsConstructor -public class DictionaryEventHandler implements EntityEventListenerCustomizer { +public class DictionaryEventHandler { + private final DefaultDictionaryItemService itemService; @@ -68,28 +73,23 @@ public class DictionaryEventHandler implements EntityEventListenerCustomizer { ); } - @Override - public void customize(EntityEventListenerConfigure configure) { - configure.enable(DictionaryItemEntity.class); - configure.enable(DictionaryEntity.class); - } - /** * 监听字典删除前事件，阻止删除分类标识为系统的字典 + * * @param event 字典删除前事件 */ @EventListener public void handleDictionaryBeforeDelete(EntityBeforeDeleteEvent event) { event.async( - Flux.fromIterable(event.getEntity()) - .any(dictionary -> - StringUtils.equals(dictionary.getClassified(), DictionaryConstants.CLASSIFIED_SYSTEM)) - .flatMap(any -> { - if (any) { - return Mono.error(() -> new BusinessException("error.system_dictionary_can_not_delete")); - } - return Mono.empty(); - }) + Flux.fromIterable(event.getEntity()) + .any(dictionary -> + StringUtils.equals(dictionary.getClassified(), DictionaryConstants.CLASSIFIED_SYSTEM)) + .flatMap(any -> { + if (any) { + return Mono.error(() -> new BusinessException("error.system_dictionary_can_not_delete")); + } + return Mono.empty(); + }) ); } } diff --git a/jetlinks-components/common-component/src/main/java/org/jetlinks/community/reactorql/EnableReactorQL.java b/jetlinks-components/common-component/src/main/java/org/jetlinks/community/reactorql/EnableReactorQL.java deleted file mode 100755 index 39d2639e..00000000 --- a/jetlinks-components/common-component/src/main/java/org/jetlinks/community/reactorql/EnableReactorQL.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.jetlinks.community.reactorql; - -import org.springframework.context.annotation.Import; - -import java.lang.annotation.*; - -/** - * 在配置类上加上此注解,并指定{@link EnableReactorQL#value()},将扫描指定包下注解了{@link ReactorQLOperation}的接口类, - * 并生成代理对象注入到spring中. - * - * @author zhouhao - * @since 1.6 - * @see ReactorQL - * @see ReactorQLOperation - */ -@Target({ElementType.TYPE}) -@Retention(RetentionPolicy.RUNTIME) -@Documented -@Inherited -@Import(ReactorQLBeanDefinitionRegistrar.class) -public @interface EnableReactorQL { - /** - * @return 扫描的包名 - */ - String[] value() default {}; -} diff --git a/jetlinks-components/common-component/src/main/java/org/jetlinks/community/reactorql/ReactorQL.java b/jetlinks-components/common-component/src/main/java/org/jetlinks/community/reactorql/ReactorQL.java deleted file mode 100755 index 178a3bf4..00000000 --- a/jetlinks-components/common-component/src/main/java/org/jetlinks/community/reactorql/ReactorQL.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.jetlinks.community.reactorql; - -import java.lang.annotation.*; - -/** - * 在接口的方法上注解,使用sql语句来处理参数 - * - * @author zhouhao - * @see org.jetlinks.reactor.ql.ReactorQL - * @see ReactorQLOperation - * @since 1.6 - */ -@Target({ElementType.METHOD, ElementType.TYPE}) -@Retention(RetentionPolicy.RUNTIME) -@Documented -@Inherited -public @interface ReactorQL { - - /** - * 使用SQL语句来处理{@link reactor.core.publisher.Flux}操作.例如分组聚合. - * 查看文档说明 - * - * 
-     *  select count(1) total,name from "arg0" group by name
-     *
- *

- *

- * 当方法有参数时,可通过arg{index}来获取参数,如: - * 

-     *     select name newName from "arg0" where id = :arg1
-     *
- * - * @return SQL语句 - */ - String[] value(); - -} diff --git a/jetlinks-components/common-component/src/main/java/org/jetlinks/community/reactorql/ReactorQLBeanDefinitionRegistrar.java b/jetlinks-components/common-component/src/main/java/org/jetlinks/community/reactorql/ReactorQLBeanDefinitionRegistrar.java deleted file mode 100755 index 01314f55..00000000 --- a/jetlinks-components/common-component/src/main/java/org/jetlinks/community/reactorql/ReactorQLBeanDefinitionRegistrar.java +++ /dev/null @@ -1,59 +0,0 @@ -package org.jetlinks.community.reactorql; - -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.support.AbstractBeanDefinition; -import org.springframework.beans.factory.support.BeanDefinitionRegistry; -import org.springframework.beans.factory.support.RootBeanDefinition; -import org.springframework.context.annotation.ImportBeanDefinitionRegistrar; -import org.springframework.context.index.CandidateComponentsIndex; -import org.springframework.context.index.CandidateComponentsIndexLoader; -import org.springframework.core.type.AnnotationMetadata; - -import javax.annotation.Nonnull; -import java.util.Arrays; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -@Slf4j -public class ReactorQLBeanDefinitionRegistrar implements ImportBeanDefinitionRegistrar { - - @Override - @SneakyThrows - public void registerBeanDefinitions(AnnotationMetadata importingClassMetadata, @Nonnull BeanDefinitionRegistry registry) { - Map attr = importingClassMetadata.getAnnotationAttributes(EnableReactorQL.class.getName()); - if (attr == null) { - return; - } - String[] packages = (String[]) attr.get("value"); - - CandidateComponentsIndex index = CandidateComponentsIndexLoader.loadIndex(org.springframework.util.ClassUtils.getDefaultClassLoader()); - if (null == index) { - return; - } - Set path = Arrays.stream(packages) - .flatMap(str -> index - .getCandidateTypes(str, ReactorQLOperation.class.getName()) - .stream()) - .collect(Collectors.toSet()); - - for (String className : path) { - Class type = org.springframework.util.ClassUtils.forName(className, null); - if (!type.isInterface() || type.getAnnotation(ReactorQLOperation.class) == null) { - continue; - } - RootBeanDefinition definition = new RootBeanDefinition(); - definition.setTargetType(type); - definition.setBeanClass(ReactorQLFactoryBean.class); - definition.setAutowireMode(AbstractBeanDefinition.AUTOWIRE_BY_TYPE); - definition.getPropertyValues().add("target", type); - if (!registry.containsBeanDefinition(type.getName())) { - log.debug("register ReactorQL Operator {}", type); - registry.registerBeanDefinition(type.getName(), definition); - } - } - - } - -} diff --git a/jetlinks-components/common-component/src/main/java/org/jetlinks/community/reactorql/ReactorQLFactoryBean.java b/jetlinks-components/common-component/src/main/java/org/jetlinks/community/reactorql/ReactorQLFactoryBean.java deleted file mode 100755 index c7b22bb7..00000000 --- a/jetlinks-components/common-component/src/main/java/org/jetlinks/community/reactorql/ReactorQLFactoryBean.java +++ /dev/null @@ -1,164 +0,0 @@ -package org.jetlinks.community.reactorql; - -import lombok.Getter; -import lombok.Setter; -import lombok.SneakyThrows; -import org.hswebframework.web.bean.FastBeanCopier; -import org.jetlinks.reactor.ql.ReactorQLContext; -import org.reactivestreams.Publisher; -import org.springframework.beans.factory.FactoryBean; -import org.springframework.beans.factory.InitializingBean; -import org.springframework.core.LocalVariableTableParameterNameDiscoverer; -import org.springframework.core.Ordered; -import org.springframework.core.ParameterNameDiscoverer; -import org.springframework.core.ResolvableType; -import org.springframework.util.ClassUtils; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; - -import java.lang.invoke.MethodHandle; -import java.lang.invoke.MethodHandles; -import java.lang.reflect.Constructor; -import java.lang.reflect.Method; -import java.lang.reflect.Proxy; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.function.Function; - -public class ReactorQLFactoryBean implements FactoryBean, InitializingBean, Ordered { - - @Getter - @Setter - private Class target; - - private Object proxy; - - private static final ParameterNameDiscoverer nameDiscoverer = new LocalVariableTableParameterNameDiscoverer(); - - public ReactorQLFactoryBean() { - - } - - @Override - public Object getObject() { - return proxy; - } - - @Override - public Class getObjectType() { - return target; - } - - @Override - public void afterPropertiesSet() { - Map> cache = new ConcurrentHashMap<>(); - - this.proxy = Proxy - .newProxyInstance(ClassUtils.getDefaultClassLoader(), - new Class[]{target}, - (proxy, method, args) -> - cache - .computeIfAbsent(method, mtd -> createInvoker(target, mtd, mtd.getAnnotation(ReactorQL.class))) - .apply(args)); - } - - @SneakyThrows - private Function createInvoker(Class type, Method method, ReactorQL ql) { - if (method.isDefault() || ql == null) { - Constructor constructor = MethodHandles.Lookup.class - .getDeclaredConstructor(Class.class); - constructor.setAccessible(true); - MethodHandles.Lookup lookup = constructor.newInstance(type); - MethodHandle handle = lookup - .in(type) - .unreflectSpecial(method, type) - .bindTo(proxy); - return args -> { - try { - return handle.invokeWithArguments(args); - } catch (Throwable e) { - return Mono.error(e); - } - }; - } - - ResolvableType returnType = ResolvableType.forMethodReturnType(method); - if (returnType.toClass() != Mono.class && returnType.toClass() != Flux.class) { - throw new UnsupportedOperationException("方法返回值必须为Mono或者Flux"); - } - Class genericType = returnType.getGeneric(0).toClass(); - Function, ?> mapper; - - if (genericType == Map.class || genericType == Object.class) { - mapper = Function.identity(); - } else { - mapper = map -> FastBeanCopier.copy(map, genericType); - } - - Function, Publisher> resultMapper = - returnType.resolve() == Mono.class - ? flux -> flux.take(1).singleOrEmpty() - : flux -> flux; - - String[] names = nameDiscoverer.getParameterNames(method); - - try { - org.jetlinks.reactor.ql.ReactorQL reactorQL = - org.jetlinks.reactor.ql.ReactorQL - .builder() - .sql(ql.value()) - .build(); - - return args -> { - Map argsMap = new HashMap<>(); - ReactorQLContext context = ReactorQLContext.ofDatasource(name -> { - if (args.length == 0) { - return Flux.just(1); - } - if (args.length == 1) { - return convertToFlux(args[0]); - } - return convertToFlux(argsMap.get(name)); - }); - for (int i = 0; i < args.length; i++) { - String indexName = "arg" + i; - - String name = names == null ? indexName : names[i]; - context.bind(i, args[i]); - context.bind(name, args[i]); - context.bind(indexName, args[i]); - argsMap.put(names == null ? indexName : names[i], args[i]); - argsMap.put(indexName, args[i]); - } - return reactorQL.start(context) - .map(record -> mapper.apply(record.asMap())) - .as(resultMapper); - }; - } catch (Throwable e) { - throw new IllegalArgumentException( - "create ReactorQL method [" + method + "] error,sql:\n" + (String.join(" ", ql.value())), e); - } - } - - protected Flux convertToFlux(Object arg) { - if (arg == null) { - return Flux.empty(); - } - if (arg instanceof Publisher) { - return Flux.from((Publisher) arg); - } - if (arg instanceof Iterable) { - return Flux.fromIterable(((Iterable) arg)); - } - if (arg instanceof Object[]) { - return Flux.fromArray(((Object[]) arg)); - } - return Flux.just(arg); - } - - @Override - public int getOrder() { - return Ordered.LOWEST_PRECEDENCE; - } -} diff --git a/jetlinks-components/common-component/src/main/java/org/jetlinks/community/reactorql/ReactorQLOperation.java b/jetlinks-components/common-component/src/main/java/org/jetlinks/community/reactorql/ReactorQLOperation.java deleted file mode 100755 index 3bcb6267..00000000 --- a/jetlinks-components/common-component/src/main/java/org/jetlinks/community/reactorql/ReactorQLOperation.java +++ /dev/null @@ -1,22 +0,0 @@ -package org.jetlinks.community.reactorql; - -import org.springframework.stereotype.Indexed; - -import java.lang.annotation.*; - -/** - * 在接口上添加此注解,开启使用sql来处理reactor数据 - * - * @author zhouhao - * @see ReactorQL - * @see EnableReactorQL - * @since 1.6 - */ -@Target({ElementType.TYPE}) -@Retention(RetentionPolicy.RUNTIME) -@Documented -@Inherited -@Indexed -public @interface ReactorQLOperation { - -} diff --git a/jetlinks-components/common-component/src/main/java/org/jetlinks/community/reactorql/term/TermValue.java b/jetlinks-components/common-component/src/main/java/org/jetlinks/community/reactorql/term/TermValue.java index 106cc84f..949feca4 100644 --- a/jetlinks-components/common-component/src/main/java/org/jetlinks/community/reactorql/term/TermValue.java +++ b/jetlinks-components/common-component/src/main/java/org/jetlinks/community/reactorql/term/TermValue.java @@ -83,7 +83,7 @@ public class TermValue implements Serializable { /** * 和manual一样, - * 兼容{@link org.jetlinks.pro.relation.utils.VariableSource.Source#fixed} + * 兼容{@link org.jetlinks.community.relation.utils.VariableSource.Source#fixed} */ fixed, manual, @@ -91,14 +91,14 @@ public class TermValue implements Serializable { metric, variable, /** - * 和variable一样,兼容{@link org.jetlinks.pro.relation.utils.VariableSource.Source#upper} + * 和variable一样,兼容{@link org.jetlinks.community.relation.utils.VariableSource.Source#upper} */ upper, /** * 函数 * - * @see org.jetlinks.pro.reactorql.function.FunctionSupport + * @see org.jetlinks.community.reactorql.function.FunctionSupport */ function } diff --git a/jetlinks-components/common-component/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/jetlinks-components/common-component/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports index c761205a..ecd312b3 100644 --- a/jetlinks-components/common-component/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports +++ b/jetlinks-components/common-component/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports @@ -1 +1,2 @@ -org.jetlinks.community.configuration.CommonConfiguration \ No newline at end of file +org.jetlinks.community.configuration.CommonConfiguration +org.jetlinks.community.dictionary.DictionaryConfiguration \ No newline at end of file diff --git a/jetlinks-components/configure-component/pom.xml b/jetlinks-components/configure-component/pom.xml index 3cd87aa0..7dd58760 100644 --- a/jetlinks-components/configure-component/pom.xml +++ b/jetlinks-components/configure-component/pom.xml @@ -5,7 +5,7 @@ jetlinks-components org.jetlinks.community - 2.3.0-SNAPSHOT + 2.10.0-SNAPSHOT ../pom.xml 4.0.0 @@ -13,8 +13,6 @@ configure-component - 8 - 8 diff --git a/jetlinks-components/configure-component/src/main/java/org/jetlinks/community/configure/JetLinksCommonConfiguration.java b/jetlinks-components/configure-component/src/main/java/org/jetlinks/community/configure/JetLinksCommonConfiguration.java index 4ff38df0..3be50661 100644 --- a/jetlinks-components/configure-component/src/main/java/org/jetlinks/community/configure/JetLinksCommonConfiguration.java +++ b/jetlinks-components/configure-component/src/main/java/org/jetlinks/community/configure/JetLinksCommonConfiguration.java @@ -1,5 +1,6 @@ package org.jetlinks.community.configure; +import org.jetlinks.community.configure.compatible.PathCompatibleFilter; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import reactor.core.scheduler.Scheduler; @@ -13,4 +14,8 @@ public class JetLinksCommonConfiguration { return Schedulers.parallel(); } + @Bean + public PathCompatibleFilter pathCompatibleFilter(){ + return new PathCompatibleFilter(); + } } diff --git a/jetlinks-components/configure-component/src/main/java/org/jetlinks/community/configure/compatible/PathCompatibleFilter.java b/jetlinks-components/configure-component/src/main/java/org/jetlinks/community/configure/compatible/PathCompatibleFilter.java new file mode 100644 index 00000000..e9b87ca3 --- /dev/null +++ b/jetlinks-components/configure-component/src/main/java/org/jetlinks/community/configure/compatible/PathCompatibleFilter.java @@ -0,0 +1,34 @@ +package org.jetlinks.community.configure.compatible; + +import org.springframework.web.server.ServerWebExchange; +import org.springframework.web.server.WebFilter; +import org.springframework.web.server.WebFilterChain; +import reactor.core.publisher.Mono; + +import javax.annotation.Nonnull; + +/** + * 兼容处理路径结尾为/的请求 + * + * @since 2.10 + */ +public class PathCompatibleFilter implements WebFilter { + @Override + @Nonnull + public Mono filter(ServerWebExchange exchange, @Nonnull WebFilterChain chain) { + String path = exchange.getRequest().getPath().toString(); + if (!path.equals("/") && path.endsWith("/")) { + path = path.substring(0, path.length() - 1); + return chain.filter( + exchange + .mutate() + .request(exchange + .getRequest() + .mutate() + .path(path) + .build()) + .build()); + } + return chain.filter(exchange); + } +} diff --git a/jetlinks-components/configure-component/src/main/java/org/jetlinks/community/configure/device/DeviceClusterConfiguration.java b/jetlinks-components/configure-component/src/main/java/org/jetlinks/community/configure/device/DeviceClusterConfiguration.java index 94e83f35..f902bb99 100644 --- a/jetlinks-components/configure-component/src/main/java/org/jetlinks/community/configure/device/DeviceClusterConfiguration.java +++ b/jetlinks-components/configure-component/src/main/java/org/jetlinks/community/configure/device/DeviceClusterConfiguration.java @@ -13,20 +13,22 @@ import org.jetlinks.core.device.session.DeviceSessionManager; import org.jetlinks.core.message.interceptor.DeviceMessageSenderInterceptor; import org.jetlinks.core.rpc.RpcManager; import org.jetlinks.core.server.MessageHandler; -import org.jetlinks.supports.cluster.ClusterDeviceOperationBroker; +import org.jetlinks.core.things.ThingRpcSupportChain; import org.jetlinks.supports.cluster.ClusterDeviceRegistry; import org.jetlinks.supports.cluster.RpcDeviceOperationBroker; -import org.jetlinks.supports.scalecube.ExtendedCluster; import org.jetlinks.supports.server.ClusterSendToDeviceMessageHandler; import org.jetlinks.supports.server.DecodedClientMessageHandler; import org.springframework.beans.BeansException; +import org.springframework.beans.factory.SmartInitializingSingleton; import org.springframework.beans.factory.config.BeanPostProcessor; +import org.springframework.boot.autoconfigure.AutoConfiguration; import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -@Configuration(proxyBeanMethods = false) +@AutoConfiguration @ConditionalOnBean(ProtocolSupports.class) public class DeviceClusterConfiguration { @@ -46,18 +48,22 @@ public class DeviceClusterConfiguration { @Bean @ConditionalOnBean(ClusterDeviceRegistry.class) - public BeanPostProcessor interceptorRegister(ClusterDeviceRegistry registry) { - return new BeanPostProcessor() { - @Override - public Object postProcessAfterInitialization(Object bean, String beanName) throws BeansException { - if (bean instanceof DeviceMessageSenderInterceptor) { - registry.addInterceptor(((DeviceMessageSenderInterceptor) bean)); - } - if (bean instanceof DeviceStateChecker) { - registry.addStateChecker(((DeviceStateChecker) bean)); - } - return bean; - } + public SmartInitializingSingleton interceptorRegister(ApplicationContext context, + ClusterDeviceRegistry registry) { + return ()->{ + + context.getBeansOfType(DeviceMessageSenderInterceptor.class) + .values() + .forEach(registry::addInterceptor); + + context.getBeansOfType(DeviceStateChecker.class) + .values() + .forEach(registry::addStateChecker); + + context.getBeansOfType(ThingRpcSupportChain.class) + .values() + .forEach(registry::addRpcChain); + }; } diff --git a/jetlinks-components/configure-component/src/main/java/org/jetlinks/community/configure/trace/TraceWebFilter.java b/jetlinks-components/configure-component/src/main/java/org/jetlinks/community/configure/trace/TraceWebFilter.java index 5236b7fc..2c2d146d 100644 --- a/jetlinks-components/configure-component/src/main/java/org/jetlinks/community/configure/trace/TraceWebFilter.java +++ b/jetlinks-components/configure-component/src/main/java/org/jetlinks/community/configure/trace/TraceWebFilter.java @@ -15,7 +15,7 @@ public class TraceWebFilter implements WebFilter, Ordered { public Mono filter(ServerWebExchange exchange, WebFilterChain chain) { // /http/method/path - String spanName = "/http/"+exchange.getRequest().getMethodValue() + exchange.getRequest().getPath().value(); + String spanName = "/http/"+exchange.getRequest().getMethod().name() + exchange.getRequest().getPath().value(); ServerHttpRequest.Builder requestCopy = exchange .getRequest() @@ -31,8 +31,12 @@ public class TraceWebFilter implements WebFilter, Ordered { //创建跟踪信息 .as(MonoTracer.create(spanName)) //从请求头中追加上级跟踪信息 - .as(MonoTracer.createWith(exchange.getRequest().getHeaders(),HttpHeadersGetter.INSTANCE)); - + .contextWrite(ctx -> { + return TraceHolder.readToContext( + ctx, + exchange.getRequest().getHeaders(), + HttpHeadersGetter.INSTANCE); + }); } @Override diff --git a/jetlinks-components/dashboard-component/pom.xml b/jetlinks-components/dashboard-component/pom.xml index dc8cce04..9b3e54c3 100644 --- a/jetlinks-components/dashboard-component/pom.xml +++ b/jetlinks-components/dashboard-component/pom.xml @@ -5,7 +5,7 @@ jetlinks-components org.jetlinks.community - 2.3.0-SNAPSHOT + 2.10.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/jetlinks-components/datasource-component/pom.xml b/jetlinks-components/datasource-component/pom.xml new file mode 100644 index 00000000..70afb063 --- /dev/null +++ b/jetlinks-components/datasource-component/pom.xml @@ -0,0 +1,74 @@ + + + + jetlinks-components + org.jetlinks.community + 2.10.0-SNAPSHOT + ../pom.xml + + 4.0.0 + + datasource-component + + + + org.springframework.boot + spring-boot-starter-data-mongodb-reactive + true + + + + org.jetlinks.community + common-component + ${project.version} + + + + com.zaxxer + HikariCP + + + + org.springframework.boot + spring-boot-starter-test + test + + + + io.r2dbc + r2dbc-spi + + + + io.r2dbc + r2dbc-pool + + + + io.r2dbc + r2dbc-h2 + test + + + + com.h2database + h2 + test + + + + org.springframework.data + spring-data-r2dbc + + + + org.jetlinks.community + rule-engine-component + ${project.version} + true + + + + \ No newline at end of file diff --git a/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/AbstractDataSource.java b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/AbstractDataSource.java new file mode 100644 index 00000000..0d68b7ce --- /dev/null +++ b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/AbstractDataSource.java @@ -0,0 +1,92 @@ +package org.jetlinks.community.datasource; + +import lombok.Generated; +import lombok.Getter; +import org.hswebframework.web.bean.FastBeanCopier; +import org.jetlinks.core.command.AbstractCommandSupport; +import reactor.core.publisher.Mono; + +import javax.annotation.Nonnull; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +@Getter +public abstract class AbstractDataSource extends AbstractCommandSupport implements DataSource { + + private final String id; + + private C config; + + private volatile boolean disposed; + + public AbstractDataSource(String id, + C config) { + this.id = id; + this.config = config; + } + + @Override + @Generated + public final String getId() { + return id; + } + + @Override + public abstract DataSourceType getType(); + + @Override + public final void dispose() { + disposed = true; + doOnDispose(); + } + + @Override + public final boolean isDisposed() { + return disposed; + } + + @Generated + public final C getConfig() { + return config; + } + + @SuppressWarnings("all") + public C copyConfig() { + return (C) FastBeanCopier.copy(config, config.getClass()); + } + + public final void setConfig(C config) { + C old = this.config; + this.config = config; + handleSetConfig(old, config); + } + + @Override + public final Mono state() { + if (isDisposed()) { + return Mono.just(DataSourceState.stopped); + } + return this.checkState(); + } + + protected Mono checkState() { + return Mono.just(DataSourceState.ok); + } + + + + protected void handleSetConfig(C oldConfig, C newConfig) { + + + } + + protected void doOnDispose() { + + } + + @Override + protected R executeUndefinedCommand(@Nonnull org.jetlinks.core.command.Command command) { + return super.executeUndefinedCommand(command); + } + +} diff --git a/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DataSource.java b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DataSource.java new file mode 100644 index 00000000..834bd669 --- /dev/null +++ b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DataSource.java @@ -0,0 +1,75 @@ +package org.jetlinks.community.datasource; + +import org.jetlinks.core.command.CommandException; +import org.jetlinks.core.command.CommandSupport; +import reactor.core.Disposable; +import reactor.core.publisher.Mono; + +import javax.annotation.Nonnull; + +/** + * 统一数据源接口定义,{@link DataSource#getType()}标识数据源类型. + *

+ * 数据源统一管理,请勿手动调用{@link DataSource#dispose()} + * + * @author zhouhao + * @since 1.10 + */ +public interface DataSource extends CommandSupport, Disposable { + + /** + * @return 数据源ID + */ + String getId(); + + /** + * @return 数据源类型 + */ + DataSourceType getType(); + + /** + * 执行指令,具体指令有对应的数据源实现定义. + * + * @param command 指令 + * @param 结果类型 + * @return void + * @see UnsupportedOperationException + */ + @Nonnull + @Override + default R execute(@Nonnull org.jetlinks.core.command.Command command) { + throw new CommandException.NoStackTrace(this, command, "error.unsupported_command"); + } + + /** + * 获取数据源状态 + * + * @return 状态 + * @see DataSourceState + */ + default Mono state() { + return Mono.just(DataSourceState.ok); + } + + /** + * 判断数据源是为指定的类型 + * + * @param target 类型 + * @return 是否为指定的类型 + */ + default boolean isWrapperFor(java.lang.Class target) { + return target.isInstance(this); + } + + /** + * 按指定类型拆箱数据源,返回对应的数据源。如果类型不一致,可能抛出{@link ClassCastException} + * + * @param target 目标类型 + * @param T + * @return 数据源 + */ + default T unwrap(Class target) { + return target.cast(this); + } + +} diff --git a/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DataSourceConfig.java b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DataSourceConfig.java new file mode 100644 index 00000000..90e9aa25 --- /dev/null +++ b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DataSourceConfig.java @@ -0,0 +1,22 @@ +package org.jetlinks.community.datasource; + +import lombok.Generated; +import lombok.Getter; +import lombok.Setter; +import org.jetlinks.community.ValueObject; + +import java.util.Map; + +@Getter +@Setter +@Generated +public class DataSourceConfig implements ValueObject { + private String id; + private String typeId; + private Map configuration; + + @Override + public Map values() { + return configuration; + } +} diff --git a/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DataSourceConfigManager.java b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DataSourceConfigManager.java new file mode 100644 index 00000000..b5bb6ae2 --- /dev/null +++ b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DataSourceConfigManager.java @@ -0,0 +1,36 @@ +package org.jetlinks.community.datasource; + +import reactor.core.Disposable; +import reactor.core.publisher.Mono; + +import java.util.function.BiFunction; + +/** + * 数据源配置管理器,统一管理数据源配置 + * + * @author zhouhao + * @since 1.10 + */ +public interface DataSourceConfigManager { + + /** + * 根据类型ID和数据源ID获取配置 + * + * @param typeId 类型ID + * @param datasourceId 数据源ID + * @return 配置信息 + */ + Mono getConfig(String typeId, String datasourceId); + + /** + * 监听配置变化，当有配置变化后将调用回调参数 + * + * @param callback 回调参数 + */ + Disposable doOnConfigChanged(BiFunction> callback); + + enum ConfigState{ + normal, + disabled + } +} diff --git a/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DataSourceConstants.java b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DataSourceConstants.java new file mode 100644 index 00000000..e24ac857 --- /dev/null +++ b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DataSourceConstants.java @@ -0,0 +1,46 @@ +package org.jetlinks.community.datasource; + +import org.jetlinks.core.command.Command; +import org.jetlinks.core.command.CommandSupport; +import org.jetlinks.core.command.CommandUtils; +import org.jetlinks.core.metadata.FunctionMetadata; +import org.jetlinks.core.metadata.SimpleFunctionMetadata; +import org.jetlinks.community.command.CommandSupportManagerProviders; +import reactor.core.publisher.Mono; + +import java.util.function.Consumer; + +public interface DataSourceConstants { + + + interface Commands { + + static String createCommandProvider(String dataSourceId) { + return "datasource$" + dataSourceId; + } + + static Mono getCommandSupport(String datasourceId) { + return CommandSupportManagerProviders + .getCommandSupport(createCommandProvider(datasourceId)); + } + + static Mono getCommandSupport(String datasourceId, String supportId) { + return CommandSupportManagerProviders + .getCommandSupport(createCommandProvider(datasourceId), supportId); + } + + } + + interface Metadata { + + static FunctionMetadata create(@SuppressWarnings("all") Class cmdType, + Consumer handler) { + SimpleFunctionMetadata metadata = new SimpleFunctionMetadata(); + metadata.setId(CommandUtils.getCommandIdByType(cmdType)); + metadata.setName(metadata.getId()); + handler.accept(metadata); + return metadata; + } + + } +} diff --git a/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DataSourceManager.java b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DataSourceManager.java new file mode 100644 index 00000000..f4fe0caa --- /dev/null +++ b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DataSourceManager.java @@ -0,0 +1,84 @@ +package org.jetlinks.community.datasource; + +import reactor.core.publisher.Flux; +import org.jetlinks.community.datasource.exception.DataSourceNotExistException; +import reactor.core.publisher.Mono; + +import java.util.List; + +/** + * 数据源管理器,用于统一管理数据源 + * + * @author zhouhao + * @since 1.10 + */ +public interface DataSourceManager { + + /** + * 获取支持的数据源类型 + * + * @return 数据源类型 + */ + List getSupportedType(); + + /** + * 根据类型获取数据眼供应商 + * + * @param typeId 类型ID + * @return 数据源供应商 + */ + DataSourceProvider getProvider(String typeId); + + + /** + * 根据类型ID获取已存在的数据源 + * + * @param typeId 类型ID + * @return 数据源列表 + */ + Flux getDataSources(String typeId); + + /** + * 获取指定的数据源,如果数据源不存在则返回{@link Mono#empty()} + * + * @param type 数据源类型 + * @param datasourceId 数据源ID + * @return 数据源 + */ + Mono getDataSource(DataSourceType type, String datasourceId); + + /** + * 获取指定的数据源,如果数据源不存在则抛出异常{@link DataSourceNotExistException} + * + * @param type 数据源类型 + * @param datasourceId 数据源ID + * @return 数据源 + * @see DataSourceNotExistException + */ + default Mono getDataSourceOrError(DataSourceType type, String datasourceId) { + return getDataSource(type, datasourceId) + .switchIfEmpty(Mono.error(() -> new DataSourceNotExistException(type, datasourceId))); + } + + /** + * 获取指定的数据源,如果数据源不存在则返回{@link Mono#empty()} + * + * @param typeId 数据源类型ID + * @param datasourceId 数据源ID + * @return 数据源 + */ + Mono getDataSource(String typeId, String datasourceId); + + /** + * 获取指定的数据源,如果数据源不存在则抛出异常{@link DataSourceNotExistException} + * + * @param typeId 数据源类型ID + * @param datasourceId 数据源ID + * @return 数据源 + * @see DataSourceNotExistException + */ + default Mono getDataSourceOrError(String typeId, String datasourceId) { + return getDataSource(typeId, datasourceId) + .switchIfEmpty(Mono.error(() -> new DataSourceNotExistException(typeId, datasourceId))); + } +} diff --git a/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DataSourceProvider.java b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DataSourceProvider.java new file mode 100644 index 00000000..adf03ace --- /dev/null +++ b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DataSourceProvider.java @@ -0,0 +1,101 @@ +package org.jetlinks.community.datasource; + +import org.jetlinks.core.command.Command; +import org.jetlinks.core.command.CommandHandler; +import org.jetlinks.core.monitor.Monitor; +import reactor.core.publisher.Mono; + +import javax.annotation.Nonnull; +import java.util.Map; + +/** + * 数据源提供商,用于提供对数据源的支持. + * + * @author zhouhao + * @see DataSource + * @since 1.10 + */ +public interface DataSourceProvider { + + /** + * @return 数据源类型 + */ + @Nonnull + DataSourceType getType(); + + /** + * 根据数据源配置来创建数据源 + * + * @param properties 数据源配置 + * @return 数据源 + */ + @Nonnull + Mono createDataSource(@Nonnull DataSourceConfig properties); + + /** + * 使用新的配置来重新加载数据源 + * + * @param dataSource 数据源 + * @param properties 配置 + * @return 重新加载后的数据源 + */ + @Nonnull + Mono reload(@Nonnull DataSource dataSource, + @Nonnull DataSourceConfig properties); + + /** + * 创建命令支持，用于提供针对某个数据源的命令支持. + *

+ * 命令执行过程中请使用{@link CommandConfiguration#getMonitor()}进行日志打印以及链路追踪。 + * + * @return 命令支持 + * @since 2.3 + */ + default Mono> createCommandHandler(CommandConfiguration configuration) { + return Mono.empty(); + } + + /** + * 命令配置 + */ + interface CommandConfiguration { + + /** + * 获取命令ID + * + * @return 命令ID + * @see Command#getCommandId() + */ + String getCommandId(); + + /** + * 获取命令名称 + * + * @return 命令名称 + */ + String getCommandName(); + + /** + * 获取命令配置信息 + * + * @return 配置信息 + */ + Map getConfiguration(); + + /** + * 获取数据源 + * + * @return 数据源 + * @see DataSource#isWrapperFor(Class) + * @see DataSource#unwrap(Class) + */ + Mono getDataSource(); + + /** + * 获取监控器,用于日志打印,链路追踪等 + * + * @return 监控器 + */ + Monitor getMonitor(); + } +} diff --git a/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DataSourceState.java b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DataSourceState.java new file mode 100644 index 00000000..3cfb2f13 --- /dev/null +++ b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DataSourceState.java @@ -0,0 +1,38 @@ +package org.jetlinks.community.datasource; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.SneakyThrows; + +@AllArgsConstructor +@Getter +public class DataSourceState { + //正常 + public static String code_ok = "ok"; + //正常 + public static String code_stopped = "stopped"; + //正常 + public static String code_error = "error"; + + public static final DataSourceState ok = new DataSourceState(code_ok, null); + public static final DataSourceState stopped = new DataSourceState(code_stopped, null); + + private final String code; + + private final Throwable reason; + + public boolean isOk(){ + return code_ok.equals(code); + } + + public static DataSourceState error(Throwable error) { + return new DataSourceState(code_error, error); + } + + @SneakyThrows + public void validate() { + if (reason != null) { + throw reason; + } + } +} diff --git a/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DataSourceType.java b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DataSourceType.java new file mode 100644 index 00000000..080391be --- /dev/null +++ b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DataSourceType.java @@ -0,0 +1,7 @@ +package org.jetlinks.community.datasource; + +public interface DataSourceType { + String getId(); + + String getName(); +} diff --git a/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DefaultDataSourceManager.java b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DefaultDataSourceManager.java new file mode 100644 index 00000000..55e264cd --- /dev/null +++ b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/DefaultDataSourceManager.java @@ -0,0 +1,235 @@ +package org.jetlinks.community.datasource; + +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Generated; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.jetlinks.core.cache.ReactiveCacheContainer; +import reactor.core.Disposable; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.util.function.Tuple2; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +/** + * 默认数据源管理器 + *

+ * 获取数据源时,如果数据源不存在. + * 则尝试从{@link DataSourceConfigManager#getConfig(String, String)}获取配置, + * 然后调用{@link DataSourceConfig#getTypeId()}对应的{@link DataSourceProvider#getType()} + * 进行初始化{@link DataSourceProvider#createDataSource(DataSourceConfig)}. + * + *

+ * 通过实现{@link DataSourceProvider}并注入到Spring容器中,即可实现自定义数据源. + * + *

+ * 当数据源配置发生变化时,将自动重新加载数据源. + * + * @author zhouhao + * @see DataSourceProvider + * @see DataSource + * @since 1.9 + */ +@Slf4j +public class DefaultDataSourceManager implements DataSourceManager { + + private final Map providers = new ConcurrentHashMap<>(); + + private final ReactiveCacheContainer cachedDataSources = ReactiveCacheContainer.create(); + + private final DataSourceConfigManager dataSourceConfigManager; + + public DefaultDataSourceManager(DataSourceConfigManager configManager) { + this.dataSourceConfigManager = configManager; + this.dataSourceConfigManager + .doOnConfigChanged((state, properties) -> { + //禁用,则删除数据源 + if (state == DataSourceConfigManager.ConfigState.disabled) { + this.removeDataSource(properties.getTypeId(), properties.getId()); + } else { + + if (cachedDataSources.containsKey(new CacheKey(properties.getTypeId(), properties.getId()))) { + //重新加载 + return this + .reloadDataSource(properties.getTypeId(), properties.getId()) + .then(); + } + + } + return Mono.empty(); + }); + } + + /** + * 注册一个数据源提供商 + * + * @param provider 数据源提供商 + * @see DataSourceProvider + */ + public void register(DataSourceProvider provider) { + log.debug("Register DataSource {} Provider {}", provider.getType().getId(), provider); + providers.put(provider.getType().getId(), provider); + } + + /** + * 注册一个已经初始化的数据源 + * + * @param dataSource 数据源 + * @see DataSource + */ + public void register(DataSource dataSource) { + log.debug("Register DataSource {} {}", dataSource.getType().getId(), dataSource); + CacheKey key = new CacheKey(dataSource.getType().getId(), dataSource.getId()); + cachedDataSources.put(key, dataSource); + } + + @Override + public DataSourceProvider getProvider(String typeId) { + DataSourceProvider dataSourceProvider = providers.get(typeId); + if (dataSourceProvider == null) { + throw new UnsupportedOperationException("不支持的数据源类型：" + typeId); + } + return dataSourceProvider; + } + + @Override + public Flux getDataSources(String typeId) { + return cachedDataSources + .values() + .filter(dataSource -> Objects.equals(typeId, dataSource.getType().getId())); + } + + @Override + public List getSupportedType() { + return providers + .values() + .stream() + .map(DataSourceProvider::getType) + .collect(Collectors.toList()); + } + + @Override + public Mono getDataSource(DataSourceType type, + String datasourceId) { + return getDataSource(type.getId(), datasourceId); + } + + @Override + public Mono getDataSource(String typeId, String datasourceId) { + return getOrCreateRef(typeId, datasourceId); + } + + /** + * 获取数据源引用缓存，如果没有则自动加载,如果数据源不存在,不会立即报错. + * 在使用{@link DataSourceRef#getRef()}才会返回错误. + * + * @param typeId 数据源类型ID + * @param datasourceId 数据源ID + * @return 数据源引用 + */ + private Mono getOrCreateRef(String typeId, String datasourceId) { + return cachedDataSources + .computeIfAbsent(new CacheKey(typeId, datasourceId), + key -> loadDataSource(key.type, key.datasourceId)); + + } + + public Mono> loadConfigAndProvider(String typeId, String datasourceId) { + return Mono + .zip( + dataSourceConfigManager.getConfig(typeId, datasourceId), + Mono + .justOrEmpty(providers.get(typeId)) + .switchIfEmpty(Mono.error(() -> new UnsupportedOperationException("unsupported datasource type " + typeId))) + ); + + } + + public Mono loadDataSource(String typeId, String datasourceId) { + return this + .loadConfigAndProvider(typeId, datasourceId) + .flatMap(tp2 -> tp2.getT2().createDataSource(tp2.getT1())) + .doOnNext(dataSource -> log + .debug("load {} datasource [{}]", dataSource.getType().getId(), dataSource.getId())); + + } + + private void removeDataSource(String typeId, String datasourceId) { + cachedDataSources.remove(new CacheKey(typeId, datasourceId)); + } + + private Mono validateDataSource(DataSourceProvider provider, DataSourceConfig config) { + + return provider + .createDataSource(config) + .flatMap(dataSource -> dataSource + .state() + .doOnNext(DataSourceState::validate) + //销毁测试数据源 + .doAfterTerminate(dataSource::dispose) + .then() + ); + } + + private Mono reloadDataSource(String typeId, String datasourceId) { + + + return this + .loadConfigAndProvider(typeId, datasourceId) + //先校验一下 + .flatMap(tp2 -> this + .validateDataSource(tp2.getT2(), tp2.getT1()) + .thenReturn(tp2)) + .flatMap(tp2 -> cachedDataSources + .compute( + new CacheKey(typeId, datasourceId), + (key, old) -> { + if (old != null) { + return tp2.getT2().reload(old, tp2.getT1()); + } + return tp2.getT2().createDataSource(tp2.getT1()); + + } + )) + .doOnError(err -> log.error("reload {} datasource [{}] error ", typeId, datasourceId, err)); + + } + + @AllArgsConstructor + @EqualsAndHashCode + static class CacheKey { + private final String type; + private final String datasourceId; + } + + static class DataSourceRef implements Disposable { + + @Getter + private volatile Mono ref; + + private boolean disposed = false; + + public DataSourceRef(Mono ref) { + this.ref = ref; + } + + @Override + public void dispose() { + ref = Mono.empty(); + disposed = true; + } + + @Override + @Generated + public boolean isDisposed() { + return disposed; + } + } + +} diff --git a/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/command/CommandHandlerProvider.java b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/command/CommandHandlerProvider.java new file mode 100644 index 00000000..a7573033 --- /dev/null +++ b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/command/CommandHandlerProvider.java @@ -0,0 +1,16 @@ +package org.jetlinks.community.datasource.command; + +import org.jetlinks.core.command.CommandHandler; +import org.jetlinks.community.datasource.DataSourceProvider; +import org.jetlinks.community.spi.Provider; +import reactor.core.publisher.Mono; + +public interface CommandHandlerProvider { + + Provider supports = Provider.create(CommandHandlerProvider.class); + + String getType(); + + Mono> createCommandHandler(DataSourceProvider.CommandConfiguration configuration); + +} diff --git a/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/command/DataSourceCommandConfig.java b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/command/DataSourceCommandConfig.java new file mode 100644 index 00000000..d5cb3d63 --- /dev/null +++ b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/command/DataSourceCommandConfig.java @@ -0,0 +1,48 @@ +package org.jetlinks.community.datasource.command; + +import lombok.Getter; +import lombok.Setter; +import org.jetlinks.core.command.Command; +import org.jetlinks.community.datasource.DataSource; + +import java.util.Map; + +@Getter +@Setter +public class DataSourceCommandConfig { + + /** + * 数据源类型 + * + * @see DataSource#getType() + */ + private String datasourceType; + /** + * 数据源ID + * + * @see DataSource#getId() + */ + private String datasourceId; + + /** + * 命令支持ID,比如一个数据源命令分类. + * + * @see org.jetlinks.community.command.CommandSupportManagerProvider#getCommandSupport(String, Map) + */ + private String supportId; + + /** + * 命令ID + * + * @see Command#getCommandId() + */ + private String commandId; + + private String commandName; + + /** + * 命令配置信息 + */ + private Map configuration; + +} diff --git a/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/command/DataSourceCommandSupportManager.java b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/command/DataSourceCommandSupportManager.java new file mode 100644 index 00000000..482218b8 --- /dev/null +++ b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/command/DataSourceCommandSupportManager.java @@ -0,0 +1,238 @@ +package org.jetlinks.community.datasource.command; + +import lombok.AllArgsConstructor; +import org.jetlinks.core.Lazy; +import org.jetlinks.core.command.*; +import org.jetlinks.core.event.EventBus; +import org.jetlinks.core.lang.SeparatedCharSequence; +import org.jetlinks.core.lang.SharedPathString; +import org.jetlinks.core.monitor.Monitor; +import org.jetlinks.core.monitor.logger.Logger; +import org.jetlinks.core.monitor.logger.Slf4jLogger; +import org.jetlinks.core.monitor.metrics.Metrics; +import org.jetlinks.core.monitor.tracer.SimpleTracer; +import org.jetlinks.core.monitor.tracer.Tracer; +import org.jetlinks.community.command.CommandSupportManagerProvider; +import org.jetlinks.community.datasource.DataSource; +import org.jetlinks.community.datasource.DataSourceConstants; +import org.jetlinks.community.datasource.DataSourceManager; +import org.jetlinks.community.datasource.DataSourceProvider; +import org.slf4j.LoggerFactory; +import org.slf4j.event.Level; +import org.springframework.util.StringUtils; +import reactor.core.Disposable; +import reactor.core.Disposables; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; + +@AllArgsConstructor +public abstract class DataSourceCommandSupportManager { + + public static final String defaultSupportId = "@@default"; + + protected final DataSourceManager dataSourceManager; + + private final Map supports = new ConcurrentHashMap<>(); + + protected Mono registerCommand(DataSourceCommandConfig config) { + return supports + .computeIfAbsent( + config.getDatasourceId(), + datasourceId -> new DataSourceCommandSupportProvider(this, datasourceId)) + .register(config); + } + + protected Mono unregisterCommand(DataSourceCommandConfig config) { + return Mono.fromRunnable(() -> { + supports.compute( + config.getDatasourceId(), + (datasourceId, + provider) -> { + if (provider != null) { + provider.unregister(config); + if (provider.isEmpty()) { + provider.dispose(); + return null; + } + } + return null; + }); + }); + } + + protected abstract Mono getCommandSupportInfo(String datasourceId, String supportId); + + static class DataSourceCommandSupportProvider + implements CommandSupportManagerProvider { + + private final DataSourceCommandSupportManager parent; + private final String datasourceId; + private final Disposable.Composite disposable = Disposables.composite(); + private final Map commandSupports = new ConcurrentHashMap<>(); + + DataSourceCommandSupportProvider(DataSourceCommandSupportManager parent, + String datasourceId) { + this.datasourceId = datasourceId; + this.parent = parent; + this.disposable.add( + CommandSupportManagerProvider + .supports + .register(this.getProvider(), this) + ); + } + + void dispose() { + this.disposable.dispose(); + } + + boolean isEmpty() { + return commandSupports.isEmpty(); + } + + //注册命令 + public Mono register(DataSourceCommandConfig config) { + return parent + .dataSourceManager + .getProvider(config.getDatasourceType()) + .createCommandHandler(new CommandConfigurationImpl(config, parent.dataSourceManager)) + .doOnNext(handler -> registerHandler(config, handler)) + .then(); + } + + + private void registerHandler(DataSourceCommandConfig config, CommandHandler handler) { + String supportId = StringUtils.hasText(config.getSupportId()) ? config.getSupportId() : defaultSupportId; + + commandSupports + .computeIfAbsent(supportId, + id -> new DataSourceCommandSupport()) + .registerHandler(config.getCommandId(), + handler); + } + + //注销命令 + public void unregister(DataSourceCommandConfig config) { + String supportId = StringUtils.hasText(config.getSupportId()) ? config.getSupportId() : defaultSupportId; + commandSupports.compute(supportId, (id, support) -> { + if (support != null) { + support.unregister(config.getCommandId()); + if (support.isEmpty()) { + return null; + } + } + return null; + }); + } + + @Override + public String getProvider() { + return DataSourceConstants.Commands.createCommandProvider(datasourceId); + } + + @Override + public Mono getCommandSupport(String id, Map options) { + if (id == null || Objects.equals(getProvider(), id)) { + id = defaultSupportId; + } + + return Mono.justOrEmpty(commandSupports.get(id)); + } + + @Override + public Flux getSupportInfo() { + return Flux + .fromIterable(commandSupports.entrySet()) + .flatMap(e -> { + String id = e.getKey(); + if (Objects.equals(id, defaultSupportId)) { + id = null; + } + return parent.getCommandSupportInfo(datasourceId, id); + }, 8); + } + + static class DataSourceCommandSupport extends AbstractCommandSupport { + @Override + protected , R> void registerHandler(String id, CommandHandler handler) { + super.registerHandler(id, handler); + } + + void unregister(String commandId) { + handlers.remove(commandId); + } + + boolean isEmpty() { + return handlers.isEmpty(); + } + } + } + + + static class CommandConfigurationImpl extends Slf4jLogger + implements DataSourceProvider.CommandConfiguration, Monitor { + private final DataSourceCommandConfig config; + private final DataSourceManager manager; + private final Tracer tracer; + + CommandConfigurationImpl(DataSourceCommandConfig config, + DataSourceManager manager) { + super(LoggerFactory.getLogger("org.jetlinks.community.datasource." + config.getDatasourceType())); + this.config = config; + this.manager = manager; + // TODO 企业版支持 链路追踪。 + this.tracer = Tracer.noop(); + } + + @Override + public String getCommandId() { + return config.getCommandId(); + } + + @Override + public String getCommandName() { + return config.getCommandName(); + } + + @Override + public Map getConfiguration() { + return config.getConfiguration(); + } + + @Override + public Mono getDataSource() { + return manager.getDataSource(config.getDatasourceType(), config.getDatasourceId()); + } + + @Override + public Monitor getMonitor() { + return this; + } + + @Override + public Logger logger() { + return this; + } + + @Override + public Tracer tracer() { + return tracer; + } + + @Override + public Metrics metrics() { + return Metrics.noop(); + } + + @Override + public void log(Level level, String message, Object... args) { + // TODO 企业版支持 页面中实时查看日志。 + super.log(level, message, args); + } + } + + +} diff --git a/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/configuration/DataSourceHandlerProviderConfiguration.java b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/configuration/DataSourceHandlerProviderConfiguration.java new file mode 100644 index 00000000..3844b9bb --- /dev/null +++ b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/configuration/DataSourceHandlerProviderConfiguration.java @@ -0,0 +1,14 @@ +package org.jetlinks.community.datasource.configuration; + +import org.springframework.boot.autoconfigure.AutoConfiguration; +import org.springframework.context.annotation.Bean; + +@AutoConfiguration +public class DataSourceHandlerProviderConfiguration { + + @Bean + public DataSourceHandlerProviderRegister dataSourceHandlerProviderRegister() { + return new DataSourceHandlerProviderRegister(); + } + +} diff --git a/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/configuration/DataSourceHandlerProviderRegister.java b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/configuration/DataSourceHandlerProviderRegister.java new file mode 100644 index 00000000..f3350bf5 --- /dev/null +++ b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/configuration/DataSourceHandlerProviderRegister.java @@ -0,0 +1,31 @@ +package org.jetlinks.community.datasource.configuration; + +import org.jetlinks.community.datasource.command.CommandHandlerProvider; +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.SmartInitializingSingleton; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; + +import javax.annotation.Nonnull; +import java.util.List; + +public class DataSourceHandlerProviderRegister implements ApplicationContextAware, SmartInitializingSingleton { + + private ApplicationContext context; + + @Override + public void setApplicationContext(@Nonnull ApplicationContext applicationContext) throws BeansException { + this.context = applicationContext; + } + + @Override + public void afterSingletonsInstantiated() { + + List commandHandlerProviders = context + .getBeanProvider(CommandHandlerProvider.class) + .stream() + .toList(); + + commandHandlerProviders.forEach(provider -> CommandHandlerProvider.supports.register(provider.getType(), provider)); + } +} diff --git a/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/configuration/DataSourceManagerConfiguration.java b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/configuration/DataSourceManagerConfiguration.java new file mode 100644 index 00000000..d1cc094c --- /dev/null +++ b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/configuration/DataSourceManagerConfiguration.java @@ -0,0 +1,24 @@ +package org.jetlinks.community.datasource.configuration; + +import org.jetlinks.community.datasource.*; +import org.jetlinks.community.datasource.*; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.boot.autoconfigure.AutoConfiguration; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.context.annotation.Bean; + +@AutoConfiguration +public class DataSourceManagerConfiguration { + + @Bean + @ConditionalOnBean(DataSourceConfigManager.class) + public DataSourceManager dataSourceManager(DataSourceConfigManager dataSourceConfigManager, + ObjectProvider providers, + ObjectProvider dataSources){ + DefaultDataSourceManager dataSourceManager= new DefaultDataSourceManager(dataSourceConfigManager); + providers.forEach(dataSourceManager::register); + dataSources.forEach(dataSourceManager::register); + return dataSourceManager; + } + +} diff --git a/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/exception/DataSourceNotExistException.java b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/exception/DataSourceNotExistException.java new file mode 100644 index 00000000..3f5dc2fd --- /dev/null +++ b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/exception/DataSourceNotExistException.java @@ -0,0 +1,15 @@ +package org.jetlinks.community.datasource.exception; + +import org.hswebframework.web.exception.I18nSupportException; +import org.jetlinks.community.datasource.DataSourceType; + +public class DataSourceNotExistException extends I18nSupportException { + + public DataSourceNotExistException(DataSourceType datasourceType, String dataSourceId) { + this(datasourceType.getId(), dataSourceId); + } + + public DataSourceNotExistException(String datasourceType, String dataSourceId) { + super("error.datasource_not_exist", datasourceType, dataSourceId); + } +} diff --git a/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/DefaultRDBDataSource.java b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/DefaultRDBDataSource.java new file mode 100644 index 00000000..933910ea --- /dev/null +++ b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/DefaultRDBDataSource.java @@ -0,0 +1,345 @@ +package org.jetlinks.community.datasource.rdb; + +import com.zaxxer.hikari.HikariDataSource; +import io.r2dbc.pool.ConnectionPool; +import io.r2dbc.pool.ConnectionPoolConfiguration; +import io.r2dbc.spi.Connection; +import lombok.SneakyThrows; +import org.apache.commons.collections4.MapUtils; +import org.hswebframework.ezorm.rdb.executor.SqlRequest; +import org.hswebframework.ezorm.rdb.executor.reactive.ReactiveSyncSqlExecutor; +import org.hswebframework.ezorm.rdb.executor.wrapper.ResultWrapper; +import org.hswebframework.ezorm.rdb.metadata.RDBDatabaseMetadata; +import org.hswebframework.ezorm.rdb.metadata.RDBSchemaMetadata; +import org.hswebframework.ezorm.rdb.operator.DatabaseOperator; +import org.hswebframework.ezorm.rdb.operator.DefaultDatabaseOperator; +import org.hswebframework.web.bean.FastBeanCopier; +import org.hswebframework.web.crud.configuration.DialectProvider; +import org.hswebframework.web.crud.query.DefaultQueryHelper; +import org.hswebframework.web.crud.query.QueryHelper; +import org.hswebframework.web.crud.sql.DefaultR2dbcExecutor; +import org.hswebframework.web.exception.I18nSupportException; +import org.jetlinks.community.datasource.rdb.command.*; +import org.jetlinks.core.command.Command; +import org.jetlinks.core.command.CommandHandler; +import org.jetlinks.core.lang.SeparatedCharSequence; +import org.jetlinks.core.lang.SharedPathString; +import org.jetlinks.community.datasource.AbstractDataSource; +import org.jetlinks.community.datasource.DataSourceState; +import org.jetlinks.community.datasource.DataSourceType; +import org.jetlinks.community.datasource.rdb.command.*; +import org.jetlinks.community.utils.ObjectMappers; +import org.reactivestreams.Publisher; +import org.springframework.boot.autoconfigure.r2dbc.R2dbcProperties; +import org.springframework.boot.context.properties.PropertyMapper; +import org.springframework.boot.r2dbc.ConnectionFactoryBuilder; +import org.springframework.dao.DataAccessResourceFailureException; +import org.springframework.r2dbc.connection.ConnectionFactoryUtils; +import org.springframework.r2dbc.connection.R2dbcTransactionManager; +import org.springframework.transaction.reactive.TransactionalOperator; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.core.publisher.Sinks; + +import javax.annotation.Nonnull; +import java.io.Closeable; +import java.sql.SQLException; +import java.time.Duration; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +public class DefaultRDBDataSource extends AbstractDataSource implements RDBDataSource { + + private DatabaseOperator operator; + + private String validateSql; + + private final QueryHelper queryHelper; + + private final List closeables = new CopyOnWriteArrayList<>(); + + private final Sinks.One loading = Sinks.one(); + + private TransactionalOperator transactionalOperator; + + public DefaultRDBDataSource(String id, + RDBDataSourceProperties config) { + super(id, config); + init(); + this.queryHelper = new DefaultQueryHelper(operator); + + //刷新RDB数据源命令：Refresh + registerHandler( + Refresh.class, + CommandHandler.of( + Refresh.metadata(), + (cmd, ignore) -> cmd.execute(operator), + Refresh::new + ) + ); + + //执行SQL命令：ExecuteSql + registerHandler( + ExecuteSql.class, + CommandHandler.of( + ExecuteSql.metadata(), + (cmd, ignore) -> loading + .asMono() + .thenMany(cmd.execute(operator)), + ExecuteSql::new + ) + ); + + //执行列表查询命令：QueryList + registerHandler( + QueryList.class, + CommandHandler.of( + QueryList.metadata(), + (cmd, ignore) -> loading + .asMono() + .thenMany(cmd.execute(operator)), + QueryList::new + ) + ); + + //执行分页查询命令：QueryPager + registerHandler( + QueryPager.class, + CommandHandler.of( + QueryPager.metadata(), + (cmd, ignore) -> loading + .asMono() + .then(cmd.execute(operator)), + QueryPager::new + ) + ); + + //执行统计数量命令：Count + registerHandler( + Count.class, + CommandHandler.of( + Count.metadata(), + (cmd, ignore) -> loading + .asMono() + .then(cmd.execute(operator)), + Count::new + ) + ); + + } + + private void loadTables() { + new Refresh().execute(operator) + .doOnTerminate(loading::tryEmitEmpty) + .subscribe(); + } + + @Override + public DatabaseOperator operator() { + return operator; + } + + @Override + public QueryHelper helper() { + return queryHelper; + } + + @Override + public DataSourceType getType() { + return RDBDataSourceType.rdb; + } + + @SneakyThrows + public void init() { + try { + if (getConfig().getType() == RDBDataSourceProperties.Type.r2dbc) { + initR2dbc(); + } else { + initJdbc(); + } + loadTables(); + validateSql = getConfig().getValidateQuery(); + } catch (Throwable e) { + throw translateException(e); + } + } + + @SneakyThrows + synchronized void initR2dbc() { + DialectProvider dialect = getConfig().dialectProvider(); + R2dbcProperties properties; + if (MapUtils.isNotEmpty(getConfig().getOthers())) { + //使用jsonCopy,FastBeanCopier不支持final字段copy. + properties = ObjectMappers + .parseJson(ObjectMappers.toJsonBytes(getConfig().getOthers()), R2dbcProperties.class); + } else { + properties = new R2dbcProperties(); + } + properties.setUrl(getConfig().getUrl()); + properties.setUsername(getConfig().getUsername()); + properties.setPassword(getConfig().getPassword()); + + PropertyMapper mapper = PropertyMapper.get().alwaysApplyingWhenNonNull(); + + ConnectionFactoryBuilder connectionFactoryBuilder = ConnectionFactoryBuilder.withUrl(properties.getUrl()); + + mapper.from(properties.getUsername()).whenNonNull().to(connectionFactoryBuilder::username); + mapper.from(properties.getPassword()).whenNonNull().to(connectionFactoryBuilder::password); + + R2dbcProperties.Pool pool = properties.getPool(); + ConnectionPoolConfiguration.Builder builder = ConnectionPoolConfiguration.builder(connectionFactoryBuilder.build()); + builder.maxLifeTime(Duration.ofMinutes(10)); + builder.maxAcquireTime(Duration.ofSeconds(10)); + mapper.from(pool.getMaxIdleTime()).to(builder::maxIdleTime); + mapper.from(pool.getInitialSize()).to(builder::initialSize); + mapper.from(pool.getMaxSize()).to(builder::maxSize); + mapper.from(pool.getValidationQuery()).whenHasText().to(builder::validationQuery); + mapper.from(pool.getMaxLifeTime()).whenNonNull().to(builder::maxLifeTime); + mapper.from(pool.getMaxAcquireTime()).whenNonNull().to(builder::maxAcquireTime); + mapper.from(pool.getMaxLifeTime()).whenNonNull().to(builder::maxLifeTime); + mapper.from(pool.getMaxAcquireTime()).whenNonNull().to(builder::maxAcquireTime); + mapper.from(pool.getValidationQuery()).whenHasText().to(builder::validationQuery); + + ConnectionPool connectionPool = new ConnectionPool(builder.build()); + closeables.add(() -> connectionPool.close().subscribe()); + transactionalOperator = TransactionalOperator.create(new R2dbcTransactionManager(connectionPool)); + + DefaultR2dbcExecutor executor = new DefaultR2dbcExecutor() { + + @Override + protected Mono getConnection() { + return ConnectionFactoryUtils.getConnection(connectionPool); + } + + @Override + public Mono execute(Publisher request) { + return super + .execute(request) + .as(transactionalOperator::transactional); + } + + @Override + public Mono update(Publisher request) { + return super + .update(request) + .as(transactionalOperator::transactional); + } + + @Override + public Flux select(Publisher request, ResultWrapper wrapper) { + return super + .select(request, wrapper) + .as(transactionalOperator::transactional); + } + }; + executor.setBindSymbol(dialect.getBindSymbol()); + executor.setBindCustomSymbol(!executor.getBindSymbol().equals("?")); + RDBDatabaseMetadata database = new RDBDatabaseMetadata(dialect.getDialect()); + database.addFeature(executor); + database.addFeature(ReactiveSyncSqlExecutor.of(executor)); + + RDBSchemaMetadata schema = dialect.createSchema(getConfig().getSchema()); + database.addSchema(schema); + database.setCurrentSchema(schema); + this.operator = DefaultDatabaseOperator.of(database); + + } + + private Throwable translateException(Throwable err) { + if (err instanceof IllegalStateException) { + if (err.getMessage() != null && err.getMessage().contains("Available drivers")) { + return new I18nSupportException("error.unsupported_database_type", err); + } + } + if (err instanceof DataAccessResourceFailureException) { + return new I18nSupportException("error.database_access_error", err); + } + if (err instanceof SQLException) { + String msg = err.getMessage(); + if (msg.contains("No suitable driver")) { + return new I18nSupportException("error.unsupported_database_type", err); + } + return err; + } + if (err.getClass() == RuntimeException.class) { + if (err.getCause() != null && err.getCause() != err) { + return translateException(err.getCause()); + } + } + return err; + } + + synchronized void initJdbc() { + HikariDataSource dataSource = new HikariDataSource(); + dataSource.setJdbcUrl(getConfig().getUrl()); + dataSource.setUsername(getConfig().getUsername()); + dataSource.setPassword(getConfig().getPassword()); + if (MapUtils.isNotEmpty(getConfig().getOthers())) { + FastBeanCopier.copy(getConfig().getOthers(), dataSource); + } + closeables.add(dataSource); + RDBDatabaseMetadata database = new RDBDatabaseMetadata(getConfig().dialectProvider().getDialect()); + database.addFeature(new RDBJdbcReactiveSqlExecutor(dataSource)); + database.addFeature(new RDBJdbcSyncSqlExecutor(dataSource)); + + RDBSchemaMetadata schema = getConfig().dialectProvider().createSchema(getConfig().getSchema()); + database.addSchema(schema); + database.setCurrentSchema(schema); + + this.operator = DefaultDatabaseOperator.of(database); + this.transactionalOperator = null; + } + + @Override + protected Mono checkState() { + return operator + .sql() + .reactive() + .select(validateSql) + .map(i -> DataSourceState.ok) + .onErrorResume(err -> Mono.just(DataSourceState.error(translateException(err)))) + .last(); + } + + @Override + @SuppressWarnings("all") + protected R executeUndefinedCommand(@Nonnull Command command) { + if (command instanceof RDBCommand) { + R r = ((RDBCommand) command).execute(operator); + if (transactionalOperator != null) { + if (r instanceof Mono) { + return (R) transactionalOperator.transactional(((Mono) r)); + } else if (r instanceof Flux) { + return (R) transactionalOperator.transactional(((Flux) r)); + } + } + return r; + } + return super.executeUndefinedCommand(command); + } + + @Override + protected void handleSetConfig(RDBDataSourceProperties oldConfig, + RDBDataSourceProperties newConfig) { + if (oldConfig == null) { + return; + } + releaseOld(); + init(); + } + + protected void releaseOld() { + closeables.removeIf(closeable -> { + try { + closeable.close(); + } catch (Throwable ignore) { + } + return true; + }); + } + + @Override + protected void doOnDispose() { + releaseOld(); + } +} diff --git a/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/RDBDataSource.java b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/RDBDataSource.java new file mode 100644 index 00000000..b4cfe4b3 --- /dev/null +++ b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/RDBDataSource.java @@ -0,0 +1,33 @@ +package org.jetlinks.community.datasource.rdb; + +import org.hswebframework.ezorm.rdb.operator.DatabaseOperator; +import org.hswebframework.web.crud.query.QueryHelper; +import org.jetlinks.community.datasource.rdb.command.*; +import org.jetlinks.community.datasource.DataSource; +import org.jetlinks.community.datasource.DataSourceType; +import org.jetlinks.core.command.Command; + +import javax.annotation.Nonnull; + + +public interface RDBDataSource extends DataSource { + + RDBDataSourceProperties getConfig(); + + RDBDataSourceProperties copyConfig(); + + DatabaseOperator operator(); + + QueryHelper helper(); + + @Nonnull + @Override + default R execute(@Nonnull Command command) { + return DataSource.super.execute(command); + } + + @Override + default DataSourceType getType() { + return RDBDataSourceType.rdb; + } +} diff --git a/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/RDBDataSourceProperties.java b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/RDBDataSourceProperties.java new file mode 100644 index 00000000..4c092446 --- /dev/null +++ b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/RDBDataSourceProperties.java @@ -0,0 +1,93 @@ +package org.jetlinks.community.datasource.rdb; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.AccessLevel; +import lombok.Getter; +import lombok.Setter; +import org.hswebframework.web.crud.configuration.DialectProvider; +import org.hswebframework.web.crud.configuration.DialectProviders; +import org.hswebframework.web.crud.configuration.EasyormProperties; +import org.hswebframework.web.exception.ValidationException; +import org.hswebframework.web.validator.ValidatorUtils; + +import jakarta.validation.constraints.NotBlank; +import java.net.URI; +import java.util.Map; + +@Getter +@Setter +public class RDBDataSourceProperties { + + private Type type; + + @Schema(description = "url") + @NotBlank + private String url; + + @NotBlank + @Schema(description = "数据库") + private String schema; + + @Schema(description = "用户名") + private String username; + + @Schema(description = "密码") + private String password; + + private Map others; + + @Schema(description = "数据库方言") + private String dialect; + + @Getter(AccessLevel.PRIVATE) + @Setter(AccessLevel.PRIVATE) + private transient DialectProvider dialectProvider; + + public String getValidateQuery() { + return dialectProvider().getValidationSql(); + } + + public DialectProvider dialectProvider() { + URI uri = URI.create(getUrl()); + + if (null == dialectProvider) { + if (null != dialect) { + dialectProvider = DialectProviders.lookup(dialect); + } else if (getUrl().contains("mysql") || getUrl().contains("mariadb")) { + return EasyormProperties.DialectEnum.mysql; + } else if (getUrl().contains("postgresql")) { + return EasyormProperties.DialectEnum.postgres; + } else if (getUrl().contains("oracle")) { + return EasyormProperties.DialectEnum.oracle; + } else if (getUrl().contains("mssql") || getUrl().contains("sqlserver")) { + return EasyormProperties.DialectEnum.mssql; + } else if (getUrl().contains("h2")) { + return EasyormProperties.DialectEnum.h2; + } else if (getUrl().contains("dm")) { + return DialectProviders.lookup("dm"); + } else { + throw new ValidationException("url", "error.unsupported_database_type", uri.getFragment()); + } + } + return dialectProvider; + } + + public RDBDataSourceProperties validate() { + ValidatorUtils.tryValidate(this); + return this; + } + + public Type getType() { + if (type == null) { + type = url.startsWith("r2dbc") ? Type.r2dbc : Type.jdbc; + } + return type; + } + + + public enum Type { + jdbc, + r2dbc; + } + +} diff --git a/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/RDBDataSourceProvider.java b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/RDBDataSourceProvider.java new file mode 100644 index 00000000..12690fee --- /dev/null +++ b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/RDBDataSourceProvider.java @@ -0,0 +1,217 @@ +package org.jetlinks.community.datasource.rdb; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import org.hswebframework.ezorm.rdb.metadata.RDBColumnMetadata; +import org.hswebframework.ezorm.rdb.operator.DatabaseOperator; +import org.hswebframework.web.api.crud.entity.QueryParamEntity; +import org.hswebframework.web.bean.FastBeanCopier; +import org.hswebframework.web.crud.query.DefaultQueryHelper; +import org.hswebframework.web.crud.query.QueryAnalyzer; +import org.hswebframework.web.crud.query.QueryHelper; +import org.jetlinks.community.datasource.DataSource; +import org.jetlinks.community.datasource.DataSourceConfig; +import org.jetlinks.community.datasource.DataSourceProvider; +import org.jetlinks.community.datasource.DataSourceType; +import org.jetlinks.core.command.CommandHandler; +import org.jetlinks.core.metadata.DataType; +import org.jetlinks.core.metadata.PropertyMetadata; +import org.jetlinks.core.metadata.SimplePropertyMetadata; +import org.jetlinks.core.metadata.types.*; +import org.jetlinks.core.monitor.Monitor; +import org.jetlinks.community.datasource.*; +import org.jetlinks.community.datasource.rdb.command.QueryList; +import org.jetlinks.community.datasource.rdb.command.QueryPager; +import org.jetlinks.community.datasource.rdb.command.RDBRequestListCommand; +import org.jetlinks.community.datasource.rdb.command.RDBRequestPagerCommand; +import org.jetlinks.community.things.utils.ThingsDatabaseUtils; +import org.jetlinks.sdk.server.commons.cmd.QueryPagerCommand; +import org.springframework.stereotype.Component; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; + +import javax.annotation.Nonnull; +import java.util.*; +import java.util.function.Function; + +@Component +public class RDBDataSourceProvider implements DataSourceProvider { + + @Nonnull + @Override + public DataSourceType getType() { + return RDBDataSourceType.rdb; + } + + @Nonnull + @Override + public Mono createDataSource(@Nonnull DataSourceConfig properties) { + + return Mono + .fromCallable(() -> RDBDataSourceProvider + .create(properties.getId(), + FastBeanCopier.copy(properties.getConfiguration(), new RDBDataSourceProperties()) + .validate() + ) + ) + .subscribeOn(Schedulers.boundedElastic()); + } + + @Nonnull + @Override + public Mono reload(@Nonnull DataSource dataSource, @Nonnull DataSourceConfig properties) { + return Mono + .defer(() -> { + RDBDataSourceProperties dataSourceProperties = FastBeanCopier + .copy(properties.getConfiguration(), RDBDataSourceProperties::new) + .validate(); + if (dataSource.isWrapperFor(DefaultRDBDataSource.class)) { + dataSource + .unwrap(DefaultRDBDataSource.class) + .setConfig(dataSourceProperties); + return Mono.just(dataSource); + } + dataSource.dispose(); + return createDataSource(properties); + }) + .subscribeOn(Schedulers.boundedElastic()); + } + + public static RDBDataSource create(String id, RDBDataSourceProperties properties) { + return new DefaultRDBDataSource(id, properties); + } + + + @Override + public Mono> createCommandHandler(CommandConfiguration configuration) { + Configuration config = FastBeanCopier.copy(configuration.getConfiguration(), Configuration.class); + RDBDefinition rdbDefinition = config.getRdbDefinition(); + Boolean paging = rdbDefinition.getPaging(); + + return getRdbDataSource(configuration) + .flatMap(source -> { + List resultColumns = getResultColumns( + rdbDefinition.getSql(), + source.helper(), + configuration.getMonitor()); + if (paging) { + return Mono.just( + RDBRequestPagerCommand + .createQueryHandler( + configuration.getCommandId(), + configuration.getCommandName(), + metadata -> metadata.setOutput(QueryPagerCommand.createOutputType(resultColumns)), + cmd -> getRdbDataSource(configuration) + .flatMap(database -> database.execute(new QueryPager().with(cmd.with("sql", rdbDefinition.getSql()).asMap()))) + ) + ); + } + return Mono.just( + RDBRequestListCommand + .createQueryHandler( + configuration.getCommandId(), + configuration.getCommandName(), + metadata -> metadata.setOutput(getResultType(resultColumns)), + cmd -> getRdbDataSource(configuration) + .flatMapMany(database -> database.execute(new QueryList().with(cmd.with("sql", rdbDefinition.getSql()).asMap()))) + )); + }); + + } + + private static Mono getRdbDataSource(CommandConfiguration configuration) { + return configuration + .getDataSource() + .map(datasource -> datasource.unwrap(RDBDataSource.class)); + } + + private Flux> queryList(String sql, DatabaseOperator operator, QueryParamEntity param) { + QueryHelper queryHelper = new DefaultQueryHelper(operator); + return queryHelper + .select(sql) + .where(param) + .fetch() + .map(Function.identity()); + } + + private DataType getResultType(List propertyMetadata) { + ObjectType type = new ObjectType(); + type.setProperties(propertyMetadata); + return type; + } + + private List getResultColumns(String sql, QueryHelper queryHelper, Monitor monitor) { + try { + QueryAnalyzer _analyzer = queryHelper.analysis(sql); + return parseColumToProperties(_analyzer); + } catch (Exception e) { + monitor + .logger() + .error("初始化sql查询失败", e); + } + return Collections.emptyList(); + } + + static List parseColumToProperties(QueryAnalyzer analyzer) { + List columns = new ArrayList<>(); + + QueryAnalyzer.Table table = analyzer.select().getTable(); + + Map> nests = new LinkedHashMap<>(); + + for (QueryAnalyzer.Column column : analyzer.select().getColumnList()) { + DataType type; + String name; + RDBColumnMetadata metadata = column.getMetadata(); + if (metadata == null) { + type = StringType.GLOBAL; + name = column.getName(); + } else { + type = ThingsDatabaseUtils.convertDataType(column.getMetadata()); + name = column.getMetadata().getComment(); + } + + if (Objects.equals(column.getOwner(), table.getAlias())) { + columns.add(SimplePropertyMetadata.of(column.getAlias(), name, type)); + } else { + nests.computeIfAbsent(column.getOwner(), __ -> new ArrayList<>()) + .add(SimplePropertyMetadata.of(column.getAlias(), name, type)); + } + } + for (Map.Entry> nest : nests.entrySet()) { + ObjectType type = new ObjectType(); + type.setProperties(nest.getValue()); + columns.add(SimplePropertyMetadata.of(nest.getKey(), nest.getKey(), type)); + } + + return columns; + } + + @Getter + @Setter + public static class Configuration { + + private RDBDefinition rdbDefinition; + + } + + @Getter + @Setter + @AllArgsConstructor + @NoArgsConstructor + public static class RDBDefinition { + + @Schema(description = "sql语句") + private String sql; + + @Schema(description = "是否分页") + private Boolean paging; + + @Schema(description = "其他配置") + private Map others; + } +} diff --git a/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/RDBDataSourceType.java b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/RDBDataSourceType.java new file mode 100644 index 00000000..e2239374 --- /dev/null +++ b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/RDBDataSourceType.java @@ -0,0 +1,18 @@ +package org.jetlinks.community.datasource.rdb; + +import org.jetlinks.community.datasource.DataSourceType; + +public enum RDBDataSourceType implements DataSourceType { + rdb + ; + + @Override + public String getId() { + return name(); + } + + @Override + public String getName() { + return "关系型数据库"; + } +} diff --git a/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/RDBJdbcReactiveSqlExecutor.java b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/RDBJdbcReactiveSqlExecutor.java new file mode 100644 index 00000000..341ddc40 --- /dev/null +++ b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/RDBJdbcReactiveSqlExecutor.java @@ -0,0 +1,58 @@ +package org.jetlinks.community.datasource.rdb; + +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.hswebframework.ezorm.rdb.executor.SqlRequest; +import org.hswebframework.ezorm.rdb.executor.jdbc.JdbcReactiveSqlExecutor; +import org.hswebframework.ezorm.rdb.executor.wrapper.ResultWrapper; +import org.reactivestreams.Publisher; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; + +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.SQLException; + +@AllArgsConstructor +@Slf4j +public class RDBJdbcReactiveSqlExecutor extends JdbcReactiveSqlExecutor { + private final DataSource dataSource; + + @Override + public Mono getConnection() { + return Mono + .using(dataSource::getConnection, + Mono::just, + source -> { + try { + source.close(); + } catch (SQLException e) { + log.error(e.getMessage(), e); + } + }, + false + ); + } + + @Override + public Mono execute(Publisher request) { + return super + .execute(request) + .subscribeOn(Schedulers.boundedElastic()); + } + + @Override + public Mono update(Publisher request) { + return super + .update(request) + .subscribeOn(Schedulers.boundedElastic()); + } + + @Override + public Flux select(Publisher request, ResultWrapper wrapper) { + return super + .select(request, wrapper) + .subscribeOn(Schedulers.boundedElastic()); + } +} diff --git a/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/RDBJdbcSyncSqlExecutor.java b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/RDBJdbcSyncSqlExecutor.java new file mode 100644 index 00000000..698afdc3 --- /dev/null +++ b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/RDBJdbcSyncSqlExecutor.java @@ -0,0 +1,26 @@ +package org.jetlinks.community.datasource.rdb; + +import lombok.AllArgsConstructor; +import lombok.SneakyThrows; +import org.hswebframework.ezorm.rdb.executor.SqlRequest; +import org.hswebframework.ezorm.rdb.executor.jdbc.JdbcSyncSqlExecutor; + +import javax.sql.DataSource; +import java.sql.Connection; + +@AllArgsConstructor +public class RDBJdbcSyncSqlExecutor extends JdbcSyncSqlExecutor { + private final DataSource dataSource; + + @Override + @SneakyThrows + public Connection getConnection(SqlRequest sqlRequest) { + return dataSource.getConnection(); + } + + @Override + @SneakyThrows + public void releaseConnection(Connection connection, SqlRequest sqlRequest) { + connection.close(); + } +} diff --git a/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/Column.java b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/Column.java new file mode 100644 index 00000000..730f6144 --- /dev/null +++ b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/Column.java @@ -0,0 +1,39 @@ +package org.jetlinks.community.datasource.rdb.command; + +import lombok.*; +import org.hswebframework.ezorm.rdb.metadata.RDBColumnMetadata; + +@Getter +@Setter +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class Column { + + private String name; + private String previousName; + + private String type; + + private boolean primaryKey; + private int length; + private int scale; + private int precision; + private boolean notnull; + + private String comment; + + public static Column of(RDBColumnMetadata columnMetadata) { + Column column = new Column(); + column.setPreviousName(columnMetadata.getName()); + column.setName(columnMetadata.getName()); + column.setType(columnMetadata.getType().getSqlType().getName()); + column.setLength(columnMetadata.getLength() == 0 ? columnMetadata.getPrecision() : columnMetadata.getLength()); + column.setScale(columnMetadata.getScale()); + column.setPrecision(columnMetadata.getPrecision()); + column.setNotnull(columnMetadata.isNotNull()); + column.setPrimaryKey(columnMetadata.isPrimaryKey()); + column.setComment(columnMetadata.getComment()); + return column; + } +} diff --git a/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/Count.java b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/Count.java new file mode 100644 index 00000000..a1c609db --- /dev/null +++ b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/Count.java @@ -0,0 +1,69 @@ +package org.jetlinks.community.datasource.rdb.command; + +import org.hswebframework.ezorm.rdb.executor.wrapper.MapResultWrapper; +import org.hswebframework.ezorm.rdb.operator.DatabaseOperator; +import org.hswebframework.web.api.crud.entity.QueryParamEntity; +import org.hswebframework.web.crud.query.DefaultQueryHelper; +import org.hswebframework.web.exception.BusinessException; +import org.jetlinks.core.metadata.FunctionMetadata; +import org.jetlinks.core.metadata.PropertyMetadata; +import org.jetlinks.community.datasource.DataSourceConstants; +import org.jetlinks.sdk.server.commons.cmd.CountCommand; +import org.springframework.data.util.Pair; +import reactor.core.publisher.Mono; + +import java.util.List; +import java.util.Optional; + +public class Count extends CountCommand implements RDBCommand> { + + public Pair getSqlOrTable() { + return Optional.ofNullable(readable().get("sql")) + .map(sql -> Pair.of("sql", String.valueOf(sql))) + .orElse(Optional.ofNullable(readable().get("table")) + .map(table -> Pair.of("table", String.valueOf(table))) + .orElse(null)); + } + + @Override + public Mono execute(DatabaseOperator operator) { + if (getSqlOrTable() == null) { + return Mono.error(new UnsupportedOperationException("sql or table is not found")); + } + + QueryParamEntity param = this.asQueryParam(); + param.setPaging(false); + DefaultQueryHelper queryHelper = new DefaultQueryHelper(operator); + + if (getSqlOrTable().getFirst().equals("sql")) { + return queryHelper + .select(getSqlOrTable().getSecond()) + .where(param) + .count(); + } + + if (getSqlOrTable().getFirst().equals("table")) { + return operator + .dml() + .query(getSqlOrTable().getSecond()) + .setParam(param) + .fetch(new MapResultWrapper()) + .reactive() + .count() + .map(Long::intValue) + .onErrorResume(err -> Mono.error(new BusinessException(err.getMessage()))); + } + + return Mono.empty(); + } + + public static FunctionMetadata metadata() { + List list = QueryList.getInputList(); + return DataSourceConstants.Metadata + .create(Count.class, func -> { + func.setName("统计数量"); + func.setInputs(list); + }); + } + +} diff --git a/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/CreateOrAlterTable.java b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/CreateOrAlterTable.java new file mode 100644 index 00000000..e2eb2e4f --- /dev/null +++ b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/CreateOrAlterTable.java @@ -0,0 +1,39 @@ +package org.jetlinks.community.datasource.rdb.command; + +import lombok.AllArgsConstructor; +import org.hswebframework.ezorm.rdb.operator.DatabaseOperator; +import org.hswebframework.ezorm.rdb.operator.ddl.TableBuilder; +import reactor.core.publisher.Mono; + +@AllArgsConstructor +public class CreateOrAlterTable implements RDBCommand> { + + private final Table table; + + @Override + public Mono execute(DatabaseOperator operator) { + + TableBuilder builder = operator + .ddl() + .createOrAlter(table.getName()); + + for (Column column : table.getColumns()) { + builder + .addColumn(column.getName()) + .custom(rdb -> { + rdb.setPrimaryKey(column.isPrimaryKey()); + rdb.setNotNull(column.isNotnull()); + rdb.setPreviousName(column.getPreviousName()); + }) + .length(column.getLength(), column.getScale()) + .type(column.getType()) + .comment(column.getComment()) + .commit(); + } + + return builder + .commit() + .reactive() + .then(); + } +} diff --git a/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/DropColumn.java b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/DropColumn.java new file mode 100644 index 00000000..93e278f8 --- /dev/null +++ b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/DropColumn.java @@ -0,0 +1,27 @@ +package org.jetlinks.community.datasource.rdb.command; + +import lombok.AllArgsConstructor; +import org.hswebframework.ezorm.rdb.operator.DatabaseOperator; +import org.hswebframework.ezorm.rdb.operator.ddl.TableBuilder; +import reactor.core.publisher.Mono; + +import java.util.Set; + +@AllArgsConstructor +public class DropColumn implements RDBCommand> { + + private final String table; + private final Set columns; + + @Override + public Mono execute(DatabaseOperator operator) { + TableBuilder builder = operator + .ddl() + .createOrAlter(table); + columns.forEach(builder::dropColumn); + return builder + .commit() + .reactive() + .then(); + } +} diff --git a/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/ExecuteSql.java b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/ExecuteSql.java new file mode 100644 index 00000000..90a2c12a --- /dev/null +++ b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/ExecuteSql.java @@ -0,0 +1,206 @@ +package org.jetlinks.community.datasource.rdb.command; + +import lombok.*; +import net.sf.jsqlparser.parser.CCJSqlParserUtil; +import net.sf.jsqlparser.statement.Statement; +import net.sf.jsqlparser.statement.select.Select; +import org.hswebframework.ezorm.rdb.executor.SqlRequest; +import org.hswebframework.ezorm.rdb.executor.SqlRequests; +import org.hswebframework.ezorm.rdb.executor.wrapper.ColumnWrapperContext; +import org.hswebframework.ezorm.rdb.executor.wrapper.ResultWrapper; +import org.hswebframework.ezorm.rdb.operator.DatabaseOperator; +import org.hswebframework.web.bean.FastBeanCopier; +import org.hswebframework.web.crud.query.DefaultQueryHelper; +import org.hswebframework.web.crud.query.QueryAnalyzer; +import org.jetlinks.community.utils.ConverterUtils; +import org.jetlinks.core.command.AbstractCommand; +import org.jetlinks.core.metadata.*; +import org.jetlinks.core.metadata.types.ArrayType; +import org.jetlinks.core.metadata.types.ObjectType; +import org.jetlinks.core.metadata.types.StringType; +import org.jetlinks.community.ConfigMetadataConstants; +import org.jetlinks.community.datasource.DataSourceConstants; +import org.springframework.util.Assert; +import reactor.core.publisher.Flux; + +import java.io.Serializable; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Consumer; + + +public class ExecuteSql extends AbstractCommand>, ExecuteSql> + implements RDBCommand>> { + + private static final String PARAMETERSTRING = "sqlRequests"; + + public List getSqlRequests() { + return ConverterUtils.convertToList(readable().getOrDefault(PARAMETERSTRING, new ExecuteSqlRequest()), + params -> FastBeanCopier.copy(params, new ExecuteSqlRequest())); + } + + @Override + public Flux> execute(DatabaseOperator operator) { + + return Flux.fromIterable(getSqlRequests()) + .flatMap(sqlRequest -> { + String sql = sqlRequest.getSql(); + Assert.hasText(sql, "'sql' can not be empty"); + SqlRequest finalRealSql = SqlRequests.template(sql, sqlRequest.getParameter()); + Statement statement = parseSql(finalRealSql.getSql()); + if (statement instanceof Select) { + return operator + .sql() + .reactive() + .select(finalRealSql, new SqlResultWrapper()) + .defaultIfEmpty(parseSelect(finalRealSql.getSql(), operator)) + .take(1000) + .collectList() + .map(list -> SqlResult.of(list, Operation.select.name())); + } + return operator + .sql() + .reactive() + .update(finalRealSql) + .map(updateCount -> SqlResult.of(updateCount, Operation.upsert.name())); + }); + } + + @AllArgsConstructor + static class SqlResultWrapper implements ResultWrapper, Void> { + + private final Map columnCountMap = new ConcurrentHashMap<>(); + + @Override + public LinkedHashMap newRowInstance() { + return new LinkedHashMap<>(); + } + + @Override + public void wrapColumn(ColumnWrapperContext> context) { + + String column = context.getColumnLabel(); + Object value = String.valueOf(context.getResult()); + + columnCountMap.compute(column, (key, count) -> { + if (context.getRowInstance().containsKey(key)) { + count = (count == null) ? 1 : count + 1; + StringBuilder sb = new StringBuilder(column) + .append("(") + .append(count) + .append(")"); + context.getRowInstance().put(sb.toString(), value); + } else { + context.getRowInstance().put(key, value); + count = 0; + } + return count; + }); + } + + @Override + public boolean completedWrapRow(LinkedHashMap result) { + if (!columnCountMap.isEmpty()) { + columnCountMap.clear(); + } + return true; + } + + @Override + public Void getResult() { + return null; + } + } + + + private LinkedHashMap parseSelect(String sql, DatabaseOperator operator) { + + QueryAnalyzer.Select select = new DefaultQueryHelper(operator) + .analysis(sql) + .select(); + + LinkedHashMap result = new LinkedHashMap<>(); + + select.getColumnList().forEach(column -> result.put(column.getName(), "(N/A)")); + + return result; + } + + @SneakyThrows + public Statement parseSql(String sql) { + return CCJSqlParserUtil.parse(sql); + } + + public static FunctionMetadata metadata(Consumer custom) { + SimpleFunctionMetadata metadata = new SimpleFunctionMetadata(); + //ExecuteSql + metadata.setId("ExecuteSql"); + metadata.setName("执行SQL语句"); + metadata.setInputs(Collections.singletonList( + SimplePropertyMetadata + .of( + PARAMETERSTRING, + "SQL查询参数", + new ArrayType().elementType( + new ObjectType() + .addProperty("sql", "sql语句", new StringType().expand(ConfigMetadataConstants.required, + true)) + .addProperty("parameter", "替换参数", new ObjectType())) + ) + )); + custom.accept(metadata); + return metadata; + } + + public static FunctionMetadata metadata() { + return DataSourceConstants.Metadata + .create(ExecuteSql.class, func -> { + func.setName("执行SQL语句"); + func.setInputs(Collections.singletonList( + SimplePropertyMetadata + .of( + PARAMETERSTRING, + "SQL查询参数", + new ArrayType().elementType( + new ObjectType() + .addProperty("sql", "sql语句", new StringType().expand(ConfigMetadataConstants.required, + true)) + .addProperty("parameter", "替换参数", new ObjectType())) + ) + )); + }); + } + + private enum Operation { + select, + upsert + } + + @Getter + @Setter + @AllArgsConstructor(staticName = "of") + public static class SqlResult { + + private T data; + + private String type; + } + + @Getter + @Setter + @AllArgsConstructor + @NoArgsConstructor + public static class ExecuteSqlRequest implements Serializable { + + private String sql; + + private Object parameter; + + } + +} + + diff --git a/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/GetTable.java b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/GetTable.java new file mode 100644 index 00000000..58d6b199 --- /dev/null +++ b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/GetTable.java @@ -0,0 +1,24 @@ +package org.jetlinks.community.datasource.rdb.command; + +import lombok.AllArgsConstructor; +import org.hswebframework.ezorm.rdb.metadata.TableOrViewMetadata; +import org.hswebframework.ezorm.rdb.metadata.parser.TableMetadataParser; +import org.hswebframework.ezorm.rdb.operator.DatabaseOperator; +import reactor.core.publisher.Mono; + +@AllArgsConstructor +public class GetTable implements RDBCommand> { + + private final String name; + + @Override + public Mono execute(DatabaseOperator operator) { + return operator + .getMetadata() + .getCurrentSchema() + .findFeatureNow(TableMetadataParser.id) + .parseByNameReactive(name) + .cast(TableOrViewMetadata.class) + .map(Table::of); + } +} diff --git a/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/GetTables.java b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/GetTables.java new file mode 100644 index 00000000..ac4ae4f2 --- /dev/null +++ b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/GetTables.java @@ -0,0 +1,29 @@ +package org.jetlinks.community.datasource.rdb.command; + +import lombok.AllArgsConstructor; +import lombok.NoArgsConstructor; +import org.hswebframework.ezorm.rdb.metadata.TableOrViewMetadata; +import org.hswebframework.ezorm.rdb.metadata.parser.TableMetadataParser; +import org.hswebframework.ezorm.rdb.operator.DatabaseOperator; +import reactor.core.publisher.Flux; + +@NoArgsConstructor +@AllArgsConstructor +public class GetTables implements RDBCommand> { + + private boolean includeColumns; + + @Override + public Flux
execute(DatabaseOperator operator) { + + TableMetadataParser parser = operator + .getMetadata() + .getCurrentSchema() + .findFeatureNow(TableMetadataParser.id); + + return + includeColumns + ? parser.parseAllReactive().cast(TableOrViewMetadata.class).map(Table::of) + : parser.parseAllTableNameReactive().map(name -> new Table(name, null)); + } +} diff --git a/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/QueryList.java b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/QueryList.java new file mode 100644 index 00000000..89fd26e5 --- /dev/null +++ b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/QueryList.java @@ -0,0 +1,77 @@ +package org.jetlinks.community.datasource.rdb.command; + +import org.hswebframework.ezorm.rdb.executor.wrapper.MapResultWrapper; +import org.hswebframework.ezorm.rdb.operator.DatabaseOperator; +import org.hswebframework.web.api.crud.entity.QueryParamEntity; +import org.hswebframework.web.crud.query.DefaultQueryHelper; +import org.jetlinks.core.metadata.FunctionMetadata; +import org.jetlinks.core.metadata.PropertyMetadata; +import org.jetlinks.core.metadata.SimplePropertyMetadata; +import org.jetlinks.core.metadata.types.StringType; +import org.jetlinks.community.datasource.DataSourceConstants; +import org.jetlinks.sdk.server.commons.cmd.QueryListCommand; +import org.springframework.data.util.Pair; +import reactor.core.publisher.Flux; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; + +public class QueryList extends QueryListCommand> + implements RDBCommand>>{ + + public Pair getSqlOrTable() { + return Optional.ofNullable(readable().get("sql")) + .map(sql -> Pair.of("sql", String.valueOf(sql))) + .orElse(Optional.ofNullable(readable().get("table")) + .map(table -> Pair.of("table", String.valueOf(table))) + .orElse(null)); + } + + @Override + public Flux> execute(DatabaseOperator operator) { + + if (getSqlOrTable() == null) { + return Flux.error(new UnsupportedOperationException("sql or table is not found")); + } + + QueryParamEntity param = this.asQueryParam(); + DefaultQueryHelper queryHelper = new DefaultQueryHelper(operator); + + if (getSqlOrTable().getFirst().equals("sql")) { + return queryHelper + .select(getSqlOrTable().getSecond()) + .where(param) + .fetch() + .map(Function.identity()); + } + + if (getSqlOrTable().getFirst().equals("table")) { + return operator + .dml() + .query(getSqlOrTable().getSecond()) + .setParam(param) + .fetch(new MapResultWrapper()) + .reactive(); + } + return Flux.empty(); + } + + public static FunctionMetadata metadata() { + List list = getInputList(); + return DataSourceConstants.Metadata + .create(QueryList.class, func -> { + func.setName("列表查询"); + func.setInputs(list); + }); + } + + public static List getInputList() { + List list = new ArrayList<>(QueryListCommand.getQueryParamMetadata()); + list.add(SimplePropertyMetadata.of("table", "表名", StringType.GLOBAL)); + list.add(SimplePropertyMetadata.of("sql", "sql语句", StringType.GLOBAL)); + return list; + } +} diff --git a/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/QueryPager.java b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/QueryPager.java new file mode 100644 index 00000000..9a8f93d9 --- /dev/null +++ b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/QueryPager.java @@ -0,0 +1,121 @@ +package org.jetlinks.community.datasource.rdb.command; + +import org.hswebframework.ezorm.rdb.executor.wrapper.MapResultWrapper; +import org.hswebframework.ezorm.rdb.operator.DMLOperator; +import org.hswebframework.ezorm.rdb.operator.DatabaseOperator; +import org.hswebframework.web.api.crud.entity.PagerResult; +import org.hswebframework.web.api.crud.entity.QueryParamEntity; +import org.hswebframework.web.crud.query.DefaultQueryHelper; +import org.hswebframework.web.crud.query.QueryHelper; +import org.jetlinks.core.command.CommandHandler; +import org.jetlinks.core.metadata.FunctionMetadata; +import org.jetlinks.core.metadata.PropertyMetadata; +import org.jetlinks.core.metadata.SimpleFunctionMetadata; +import org.jetlinks.core.metadata.SimplePropertyMetadata; +import org.jetlinks.core.metadata.types.StringType; +import org.jetlinks.community.datasource.DataSourceConstants; +import org.jetlinks.sdk.server.commons.cmd.QueryPagerCommand; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.util.*; +import java.util.function.Consumer; +import java.util.function.Function; + + +public class QueryPager extends QueryPagerCommand> + implements RDBCommand>>> { + + private final static String PAGING_COLUMN = "ROWNUM_"; + + public String getTable() { + return String.valueOf(readable().get("table")); + } + + public String getSql() { + return String.valueOf(readable().get("sql")); + } + + public QueryParamEntity getParam() { + return this.asQueryParam(); + } + + @Override + public Mono>> execute(DatabaseOperator operator) { + + QueryParamEntity param = getParam(); + String table = getTable(); + String sql = getSql(); + DMLOperator dmlOperator = operator.dml(); + + DefaultQueryHelper queryHelper = new DefaultQueryHelper(operator); + + if (sql.isEmpty() || sql.equals("null")) { + return Mono.zip( + dmlOperator + .query(table) + .setParam(param) + .paging(param.getPageIndex(), param.getPageSize()) + .fetch(new MapResultWrapper()) + .reactive() + .map(map -> { + map.remove(PAGING_COLUMN); + return map; + }) + .collectList(), + dmlOperator + .createReactiveRepository(table) + .createQuery() + .setParam(param) + .count(), + (list, total) -> PagerResult.of(total, list, param)); + } + return QueryHelper + .transformPageResult( + queryHelper + .select(sql) + .where(param) + .fetchPaged(), + list -> Flux + .fromIterable(list) + .map(record -> (Map)record) + .collectList() + ); + } + + public static FunctionMetadata metadata() { + List list = getInputList(); + return DataSourceConstants.Metadata + .create(QueryPager.class, func -> { + func.setName("分页查询"); + func.setInputs(list); + }); + } + + public static List getInputList() { + List list = new ArrayList<>(QueryPagerCommand.getQueryParamMetadata()); + list.add(SimplePropertyMetadata.of("table", "表名", StringType.GLOBAL)); + list.add(SimplePropertyMetadata.of("sql", "sql语句", StringType.GLOBAL)); + return list; + } + + public static CommandHandler>>> createQueryHandler( + Consumer custom, + Function>>> handler) { + return CommandHandler.of( + () -> metadata(custom), + (cmd, ignore) -> handler.apply(cmd), + QueryPager::new + ); + + } + + public static FunctionMetadata metadata(Consumer custom) { + SimpleFunctionMetadata metadata = new SimpleFunctionMetadata(); + metadata.setId("QueryPager"); + metadata.setName("分页查询"); + metadata.setInputs(getInputList()); + custom.accept(metadata); + return metadata; + } +} diff --git a/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/RDBCommand.java b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/RDBCommand.java new file mode 100644 index 00000000..16cbd6a5 --- /dev/null +++ b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/RDBCommand.java @@ -0,0 +1,9 @@ +package org.jetlinks.community.datasource.rdb.command; + +import org.hswebframework.ezorm.rdb.operator.DatabaseOperator; +import org.jetlinks.core.command.Command; + +public interface RDBCommand extends Command { + + T execute(DatabaseOperator operator); +} diff --git a/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/RDBRequestListCommand.java b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/RDBRequestListCommand.java new file mode 100644 index 00000000..31ff5815 --- /dev/null +++ b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/RDBRequestListCommand.java @@ -0,0 +1,65 @@ +package org.jetlinks.community.datasource.rdb.command; + +import org.hswebframework.web.bean.FastBeanCopier; +import org.jetlinks.core.command.CommandHandler; +import org.jetlinks.core.metadata.FunctionMetadata; +import org.jetlinks.core.metadata.PropertyMetadata; +import org.jetlinks.core.metadata.SimpleFunctionMetadata; +import org.jetlinks.sdk.server.commons.cmd.QueryListCommand; +import reactor.core.publisher.Flux; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; +import java.util.function.Function; + + +public class RDBRequestListCommand extends QueryListCommand> { + + private String commandId; + + @Override + public String getCommandId() { + return commandId; + } + + public RDBRequestListCommand(String commandId) { + this.commandId = commandId; + withConverter(RDBRequestListCommand::convertMap); + } + + public RDBRequestListCommand() { + withConverter(RDBRequestListCommand::convertMap); + } + + + + public static CommandHandler>> createQueryHandler( + String commandId, + String commandName, + Consumer custom, + Function>> handler) { + return CommandHandler.of( + () -> metadata(commandId, commandName, custom), + (cmd, ignore) -> handler.apply(cmd), + () -> new RDBRequestListCommand(commandId) + ); + + } + + public static FunctionMetadata metadata(String commandId, String commandName, Consumer custom) { + SimpleFunctionMetadata metadata = new SimpleFunctionMetadata(); + List queryParamMetadata = QueryListCommand.getQueryParamMetadata(); + metadata.setId(commandId); + metadata.setName(commandName); + metadata.setInputs(queryParamMetadata); + custom.accept(metadata); + return metadata; + } + + public static Map convertMap(Object obj) { + return obj instanceof Map ? (Map) obj : FastBeanCopier.copy(obj, new HashMap<>()); + } + +} diff --git a/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/RDBRequestPagerCommand.java b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/RDBRequestPagerCommand.java new file mode 100644 index 00000000..8eed3116 --- /dev/null +++ b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/RDBRequestPagerCommand.java @@ -0,0 +1,58 @@ +package org.jetlinks.community.datasource.rdb.command; + +import org.hswebframework.web.api.crud.entity.PagerResult; +import org.jetlinks.core.command.CommandHandler; +import org.jetlinks.core.metadata.FunctionMetadata; +import org.jetlinks.core.metadata.PropertyMetadata; +import org.jetlinks.core.metadata.SimpleFunctionMetadata; +import org.jetlinks.sdk.server.commons.cmd.QueryPagerCommand; +import reactor.core.publisher.Mono; + +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; +import java.util.function.Function; + + +public class RDBRequestPagerCommand extends QueryPagerCommand> { + + private String commandId; + + @Override + public String getCommandId() { + return commandId; + } + + public RDBRequestPagerCommand(String commandId) { + this.commandId = commandId; + withConverter(RDBRequestListCommand::convertMap); + } + + public RDBRequestPagerCommand() { + withConverter(RDBRequestListCommand::convertMap); + } + + public static CommandHandler>>> createQueryHandler( + String commandId, + String commandName, + Consumer custom, + Function>>> handler) { + return CommandHandler.of( + () -> metadata(commandId, commandName, custom), + (cmd, ignore) -> handler.apply(cmd), + () -> new RDBRequestPagerCommand(commandId) + ); + + } + + public static FunctionMetadata metadata(String commandId, String commandName,Consumer custom) { + SimpleFunctionMetadata metadata = new SimpleFunctionMetadata(); + List queryParamMetadata = QueryPagerCommand.getQueryParamMetadata(); + metadata.setId(commandId); + metadata.setName(commandName); + metadata.setInputs(queryParamMetadata); + custom.accept(metadata); + return metadata; + } + +} diff --git a/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/Refresh.java b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/Refresh.java new file mode 100644 index 00000000..865909b4 --- /dev/null +++ b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/Refresh.java @@ -0,0 +1,30 @@ +package org.jetlinks.community.datasource.rdb.command; + +import lombok.AllArgsConstructor; +import org.hswebframework.ezorm.rdb.metadata.RDBSchemaMetadata; +import org.hswebframework.ezorm.rdb.operator.DatabaseOperator; +import org.jetlinks.core.metadata.FunctionMetadata; +import org.jetlinks.community.datasource.DataSourceConstants; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +@AllArgsConstructor +public class Refresh implements RDBCommand> { + + @Override + public Mono execute(DatabaseOperator operator) { + return Flux + .fromIterable(operator + .getMetadata() + .getSchemas()) + .concatMap(RDBSchemaMetadata::loadAllTableReactive) + .then(); + } + + public static FunctionMetadata metadata() { + return DataSourceConstants.Metadata + .create(Refresh.class, func -> { + func.setName("刷新RDB数据源信息"); + }); + } +} diff --git a/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/Table.java b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/Table.java new file mode 100644 index 00000000..c5d37a78 --- /dev/null +++ b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/Table.java @@ -0,0 +1,27 @@ +package org.jetlinks.community.datasource.rdb.command; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import org.hswebframework.ezorm.rdb.metadata.TableOrViewMetadata; + +import java.util.List; +import java.util.stream.Collectors; + +@Getter +@Setter +@AllArgsConstructor +@NoArgsConstructor +public class Table { + private String name; + + private List columns; + + public static Table of(TableOrViewMetadata metadata){ + Table table = new Table(); + table.setName(metadata.getName()); + table.setColumns(metadata.getColumns().stream().map(Column::of).collect(Collectors.toList())); + return table; + } +} diff --git a/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/Upsert.java b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/Upsert.java new file mode 100644 index 00000000..61da2b63 --- /dev/null +++ b/jetlinks-components/datasource-component/src/main/java/org/jetlinks/community/datasource/rdb/command/Upsert.java @@ -0,0 +1,39 @@ +package org.jetlinks.community.datasource.rdb.command; + +import lombok.AllArgsConstructor; +import org.hswebframework.ezorm.rdb.operator.DatabaseOperator; +import reactor.core.publisher.Mono; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; + +@AllArgsConstructor +public class Upsert implements RDBCommand> { + private final String table; + + private final List> dataList; + + private final Set ignoreUpdateColumn; + + public Upsert(String table, List> dataList) { + this(table, dataList, Collections.emptySet()); + } + + @Override + public Mono execute(DatabaseOperator operator) { + return operator + .getMetadata() + .getTableReactive(table) + .flatMap(tableMetadata -> operator + .dml() + .upsert(tableMetadata) + .values(dataList) + .ignoreUpdate(ignoreUpdateColumn == null ? new String[0] : ignoreUpdateColumn.toArray(new String[0])) + .execute() + .reactive() + .then() + ); + } +} diff --git a/jetlinks-components/datasource-component/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/jetlinks-components/datasource-component/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports new file mode 100644 index 00000000..ac133824 --- /dev/null +++ b/jetlinks-components/datasource-component/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports @@ -0,0 +1,2 @@ +org.jetlinks.community.datasource.configuration.DataSourceManagerConfiguration +org.jetlinks.community.datasource.configuration.DataSourceHandlerProviderConfiguration \ No newline at end of file diff --git a/jetlinks-components/datasource-component/src/main/resources/i18n/datasource/messages_en.properties b/jetlinks-components/datasource-component/src/main/resources/i18n/datasource/messages_en.properties new file mode 100644 index 00000000..9b32e24e --- /dev/null +++ b/jetlinks-components/datasource-component/src/main/resources/i18n/datasource/messages_en.properties @@ -0,0 +1,3 @@ +error.datasource_not_exist=The datasource {0}:{1} does not exist +error.unsupported_database_type=Unsupported database type +error.database_access_error=Database access error \ No newline at end of file diff --git a/jetlinks-components/datasource-component/src/main/resources/i18n/datasource/messages_zh.properties b/jetlinks-components/datasource-component/src/main/resources/i18n/datasource/messages_zh.properties new file mode 100644 index 00000000..acf0099d --- /dev/null +++ b/jetlinks-components/datasource-component/src/main/resources/i18n/datasource/messages_zh.properties @@ -0,0 +1,3 @@ +error.datasource_not_exist=\u6570\u636E\u6E90{0}:{1}\u4E0D\u5B58\u5728 +error.unsupported_database_type=\u4E0D\u652F\u6301\u7684\u6570\u636E\u5E93\u7C7B\u578B +error.database_access_error=\u6570\u636E\u5E93\u7528\u6237\u540D\u6216\u5BC6\u7801\u4E0D\u6B63\u786E \ No newline at end of file diff --git a/jetlinks-components/elasticsearch-component/.gitignore b/jetlinks-components/elasticsearch-component/.gitignore new file mode 100755 index 00000000..82f0c3ac --- /dev/null +++ b/jetlinks-components/elasticsearch-component/.gitignore @@ -0,0 +1 @@ +/data/ diff --git a/jetlinks-components/elasticsearch-component/elasticsearch-7x/pom.xml b/jetlinks-components/elasticsearch-component/elasticsearch-7x/pom.xml new file mode 100644 index 00000000..0fbc2049 --- /dev/null +++ b/jetlinks-components/elasticsearch-component/elasticsearch-7x/pom.xml @@ -0,0 +1,42 @@ + + + 4.0.0 + + org.jetlinks.community + elasticsearch-component + 2.10.0-SNAPSHOT + ../pom.xml + + + elasticsearch-7x + + + + + org.jetlinks.community + elasticsearch-core + ${project.version} + + + + co.elastic.clients + elasticsearch-java + ${elasticsearch7x.version} + + + org.elasticsearch.client + elasticsearch-rest-client + ${elasticsearch7x.version} + + + + + org.jetlinks.community + things-component + true + + + + \ No newline at end of file diff --git a/jetlinks-components/elasticsearch-component/elasticsearch-7x/src/main/java/org/jetlinks/community/elastic/search/ElasticSearch7xSupport.java b/jetlinks-components/elasticsearch-component/elasticsearch-7x/src/main/java/org/jetlinks/community/elastic/search/ElasticSearch7xSupport.java new file mode 100644 index 00000000..210eec89 --- /dev/null +++ b/jetlinks-components/elasticsearch-component/elasticsearch-7x/src/main/java/org/jetlinks/community/elastic/search/ElasticSearch7xSupport.java @@ -0,0 +1,67 @@ +package org.jetlinks.community.elastic.search; + +import co.elastic.clients.elasticsearch._types.aggregations.DateHistogramBucket; +import co.elastic.clients.elasticsearch._types.aggregations.HistogramBucket; +import co.elastic.clients.elasticsearch._types.aggregations.MultiBucketBase; +import co.elastic.clients.elasticsearch._types.mapping.DynamicTemplate; +import co.elastic.clients.elasticsearch._types.mapping.Property; +import co.elastic.clients.elasticsearch.indices.*; +import co.elastic.clients.elasticsearch.indices.get_mapping.IndexMappingRecord; +import co.elastic.clients.transport.Version; +import org.jetlinks.community.elastic.search.enums.ElasticSearchTermTypes; +import org.jetlinks.community.elastic.search.enums.ElasticSearch7xTermType; +import org.jetlinks.community.elastic.search.index.ElasticSearchIndexProperties; + +public class ElasticSearch7xSupport extends ElasticSearchSupport { + + static { + Version version = Version.VERSION; + if (version != null && version.major() == 7) { + for (ElasticSearch7xTermType value : ElasticSearch7xTermType.values()) { + ElasticSearchTermTypes.register(value); + } + } + } + + @Override + public IndexSettings.Builder applyIndexSettings(ElasticSearchIndexProperties index, IndexSettings.Builder builder) { + return super + .applyIndexSettings(index, builder) + .mapping(b -> b + .totalFields(t -> t.limit((int) index.getTotalFieldsLimit()))); + } + + @Override + public DynamicTemplate createDynamicTemplate(String type, Property property) { + return DynamicTemplate + .of(b -> b + .matchMappingType(type) + .mapping(property)); + } + + @Override + public TemplateMapping getTemplateMapping(GetTemplateResponse response, String index) { + return response.get(index); + } + + @Override + public IndexState getIndexState(GetIndexResponse response, String index) { + return response.get(index); + } + + @Override + public IndexMappingRecord getIndexMapping(GetMappingResponse response, String index) { + return response.get(index); + } + + @Override + protected Object getBucketKey(MultiBucketBase bucket) { + if (bucket instanceof DateHistogramBucket _bucket) { + return _bucket.key(); + } + if (bucket instanceof HistogramBucket _bucket) { + return _bucket.key(); + } + return null; + } +} diff --git a/jetlinks-components/elasticsearch-component/elasticsearch-7x/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticSearch7xTermType.java b/jetlinks-components/elasticsearch-component/elasticsearch-7x/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticSearch7xTermType.java new file mode 100755 index 00000000..d9c04c82 --- /dev/null +++ b/jetlinks-components/elasticsearch-component/elasticsearch-7x/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticSearch7xTermType.java @@ -0,0 +1,232 @@ +package org.jetlinks.community.elastic.search.enums; + +import co.elastic.clients.elasticsearch._types.FieldValue; +import co.elastic.clients.elasticsearch._types.query_dsl.Query; +import co.elastic.clients.json.JsonData; +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.hswebframework.ezorm.core.param.Term; +import org.hswebframework.ezorm.core.param.TermType; +import org.jetlinks.core.metadata.DataType; +import org.jetlinks.community.utils.ConverterUtils; +import org.jetlinks.reactor.ql.utils.CastUtils; +import org.springframework.util.ObjectUtils; + +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Optional; + +/** + * @author Jia_RG , bestfeng ,zhouhao + */ +@Getter +@AllArgsConstructor +public enum ElasticSearch7xTermType implements ElasticSearchTermType { + eq(TermType.eq) { + @Override + public Query.Builder process(Term term, Query.Builder builder) { + + builder.term(termQuery -> termQuery + .field(term.getColumn().trim()) + .value(FieldValue.of(JsonData.of(term.getValue())))); + + return builder; + } + }, + not(TermType.not) { + @Override + public Query.Builder process(Term term, Query.Builder builder) { + builder.bool(bool -> bool + .mustNot(mustNot -> mustNot + .term(termQuery -> termQuery + .field(term.getColumn().trim()) + .value(FieldValue.of(JsonData.of(term.getValue())))))); + return builder; + } + }, + isnull(TermType.isnull) { + @Override + public Query.Builder process(Term term, Query.Builder builder) { + builder + .bool(bool -> bool + .mustNot(mustNot -> notnull.process(term, mustNot))); + return builder; + } + + @Override + public Object convertTermValue(DataType type, Object value) { + return value; + } + }, + notnull(TermType.notnull) { + @Override + public Query.Builder process(Term term, Query.Builder builder) { + builder.exists(exists -> exists.field(term.getColumn())); + return builder; + } + + @Override + public Object convertTermValue(DataType type, Object value) { + return value; + } + }, + empty(TermType.empty) { + @Override + public Query.Builder process(Term term, Query.Builder builder) { + builder.term(t -> t.field(term.getColumn().trim()).value("")); + return builder; + } + + @Override + public Object convertTermValue(DataType type, Object value) { + return value; + } + }, + nempty(TermType.nempty) { + @Override + public Query.Builder process(Term term, Query.Builder builder) { + builder.bool(bool -> bool + .mustNot(mustNot -> empty.process(term, mustNot))); + return builder; + } + + @Override + public Object convertTermValue(DataType type, Object value) { + return value; + } + }, + btw(TermType.btw) { + @Override + public Query.Builder process(Term term, Query.Builder builder) { + Number between = null; + Number and = null; + List values = ConverterUtils.convertToList(term.getValue()); + int size = values.size(); + if (size > 0) { + between = CastUtils.castNumber(values.get(0)); + } + if (size > 1) { + and = CastUtils.castNumber(values.get(1)); + } + Number fb = between, ab = and; + builder.range(range -> range + .field(term.getColumn()) + .gte(JsonData.of(fb)) + .lte(JsonData.of(ab))); + return builder; + } + }, + gt(TermType.gt) { + @Override + public Query.Builder process(Term term, Query.Builder builder) { + Number value = CastUtils.castNumber(term.getValue()); + builder.range(range -> range + .field(term.getColumn()) + .gt(JsonData.of(value))); + return builder; + } + }, + gte(TermType.gte) { + @Override + public Query.Builder process(Term term, Query.Builder builder) { + Number value = CastUtils.castNumber(term.getValue()); + builder.range(range -> range + .field(term.getColumn()) + .gte(JsonData.of(value))); + return builder; + } + }, + lt(TermType.lt) { + @Override + public Query.Builder process(Term term, Query.Builder builder) { + Number value = CastUtils.castNumber(term.getValue()); + builder.range(range -> range + .field(term.getColumn()) + .lt(JsonData.of(value))); + return builder; + } + }, + lte(TermType.lte) { + @Override + public Query.Builder process(Term term, Query.Builder builder) { + Number value = CastUtils.castNumber(term.getValue()); + builder.range(range -> range + .field(term.getColumn()) + .lte(JsonData.of(value))); + return builder; + } + }, + in(TermType.in) { + @Override + public Query.Builder process(Term term, Query.Builder builder) { + builder.terms(termQuery -> termQuery + .field(term.getColumn().trim()) + .terms(t -> t.value( + ConverterUtils.convertToList(term.getValue(), d->FieldValue.of(JsonData.of(d))) + ))); + return builder; + } + }, + nin(TermType.nin) { + @Override + public Query.Builder process(Term term, Query.Builder builder) { + builder.bool(bool -> bool + .mustNot(mustNot -> in.process(term, mustNot))); + return builder; + } + }, + like(TermType.like) { + @Override + public Query.Builder process(Term term, Query.Builder builder) { + builder.wildcard(wildcard -> wildcard + .field(term.getColumn().trim()) + .value(likeQueryTermValueHandler(term.getValue()))); + return builder; + } + }, + nlike(TermType.nlike) { + @Override + public Query.Builder process(Term term, Query.Builder builder) { + builder.bool(bool -> bool + .mustNot(mustNot -> mustNot + .wildcard(wildcard -> wildcard + .field(term.getColumn().trim()) + .value(likeQueryTermValueHandler(term.getValue()))))); + return builder; + } + }; + + private final String type; + + @Override + public boolean isSupported(Term term) { + return this.type.equalsIgnoreCase(term.getTermType()); + } + + @Override + public String getId() { + return type; + } + + private static String convertString(Object value) { + if (value instanceof Collection) { + return String.join(",", ((Collection) value)); + } else { + return String.valueOf(value); + } + } + + public static String likeQueryTermValueHandler(Object value) { + if (!ObjectUtils.isEmpty(value)) { + return convertString(value).replace("%", "*"); + } + return "**"; + } + + public static Optional of(String type) { + return Arrays.stream(values()) + .filter(e -> e.getType().equalsIgnoreCase(type)) + .findAny(); + } +} diff --git a/jetlinks-components/elasticsearch-component/elasticsearch-7x/src/main/resources/META-INF/services/org.jetlinks.community.elastic.search.ElasticSearchSupport b/jetlinks-components/elasticsearch-component/elasticsearch-7x/src/main/resources/META-INF/services/org.jetlinks.community.elastic.search.ElasticSearchSupport new file mode 100644 index 00000000..2d052d6b --- /dev/null +++ b/jetlinks-components/elasticsearch-component/elasticsearch-7x/src/main/resources/META-INF/services/org.jetlinks.community.elastic.search.ElasticSearchSupport @@ -0,0 +1 @@ +org.jetlinks.community.elastic.search.ElasticSearch7xSupport \ No newline at end of file diff --git a/jetlinks-components/elasticsearch-component/elasticsearch-8x/pom.xml b/jetlinks-components/elasticsearch-component/elasticsearch-8x/pom.xml new file mode 100644 index 00000000..52d015a6 --- /dev/null +++ b/jetlinks-components/elasticsearch-component/elasticsearch-8x/pom.xml @@ -0,0 +1,40 @@ + + + 4.0.0 + + org.jetlinks.community + elasticsearch-component + 2.10.0-SNAPSHOT + ../pom.xml + + + elasticsearch-8x + + + + + + org.jetlinks.community + elasticsearch-core + ${project.version} + + + + co.elastic.clients + elasticsearch-java + + + org.elasticsearch.client + elasticsearch-rest-client + + + + org.jetlinks.community + things-component + test + + + + \ No newline at end of file diff --git a/jetlinks-components/elasticsearch-component/elasticsearch-8x/src/main/java/org/jetlinks/community/elastic/search/ElasticSearch8xSupport.java b/jetlinks-components/elasticsearch-component/elasticsearch-8x/src/main/java/org/jetlinks/community/elastic/search/ElasticSearch8xSupport.java new file mode 100644 index 00000000..400af1db --- /dev/null +++ b/jetlinks-components/elasticsearch-component/elasticsearch-8x/src/main/java/org/jetlinks/community/elastic/search/ElasticSearch8xSupport.java @@ -0,0 +1,73 @@ +package org.jetlinks.community.elastic.search; + +import co.elastic.clients.elasticsearch._types.aggregations.DateHistogramBucket; +import co.elastic.clients.elasticsearch._types.aggregations.HistogramBucket; +import co.elastic.clients.elasticsearch._types.aggregations.MultiBucketBase; +import co.elastic.clients.elasticsearch._types.mapping.DynamicTemplate; +import co.elastic.clients.elasticsearch._types.mapping.Property; +import co.elastic.clients.elasticsearch.indices.*; +import co.elastic.clients.elasticsearch.indices.get_mapping.IndexMappingRecord; +import co.elastic.clients.transport.Version; +import org.jetlinks.community.elastic.search.enums.ElasticSearchTermTypes; +import org.jetlinks.community.elastic.search.enums.ElasticSearch8xTermType; +import org.jetlinks.community.elastic.search.index.ElasticSearchIndexProperties; + +public class ElasticSearch8xSupport extends ElasticSearchSupport { + + static { + Version version = Version.VERSION; + if (version != null && version.major() == 8) { + for (ElasticSearch8xTermType value : ElasticSearch8xTermType.values()) { + ElasticSearchTermTypes.register(value); + } + } + + } + + @Override + public boolean is8x() { + return true; + } + + @Override + public IndexSettings.Builder applyIndexSettings(ElasticSearchIndexProperties index, IndexSettings.Builder builder) { + return super + .applyIndexSettings(index, builder) + .mapping(b -> b + .totalFields(t -> t.limit(index.getTotalFieldsLimit()))); + } + + @Override + public DynamicTemplate createDynamicTemplate(String type, Property property) { + return DynamicTemplate + .of(b -> b + .matchMappingType(type) + .mapping(property)); + } + + @Override + public TemplateMapping getTemplateMapping(GetTemplateResponse response, String index) { + return response.get(index); + } + + @Override + public IndexState getIndexState(GetIndexResponse response, String index) { + return response.get(index); + } + + @Override + public IndexMappingRecord getIndexMapping(GetMappingResponse response, String index) { + return response.get(index); + } + + @Override + protected Object getBucketKey(MultiBucketBase bucket) { + if (bucket instanceof DateHistogramBucket _bucket) { + return _bucket.key(); + } + if (bucket instanceof HistogramBucket _bucket) { + return _bucket.key(); + } + return null; + } +} diff --git a/jetlinks-components/elasticsearch-component/elasticsearch-8x/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticSearch8xTermType.java b/jetlinks-components/elasticsearch-component/elasticsearch-8x/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticSearch8xTermType.java new file mode 100755 index 00000000..b26e989f --- /dev/null +++ b/jetlinks-components/elasticsearch-component/elasticsearch-8x/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticSearch8xTermType.java @@ -0,0 +1,238 @@ +package org.jetlinks.community.elastic.search.enums; + +import co.elastic.clients.elasticsearch._types.FieldValue; +import co.elastic.clients.elasticsearch._types.query_dsl.Query; +import co.elastic.clients.json.JsonData; +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.hswebframework.ezorm.core.param.Term; +import org.hswebframework.ezorm.core.param.TermType; +import org.jetlinks.core.metadata.DataType; +import org.jetlinks.community.utils.ConverterUtils; +import org.jetlinks.reactor.ql.utils.CastUtils; +import org.springframework.util.ObjectUtils; + +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Optional; + +/** + * @author Jia_RG , bestfeng ,zhouhao + */ +@Getter +@AllArgsConstructor +public enum ElasticSearch8xTermType implements ElasticSearchTermType { + eq(TermType.eq) { + @Override + public Query.Builder process(Term term, Query.Builder builder) { + + builder.term(termQuery -> termQuery + .field(term.getColumn().trim()) + .value(FieldValue.of(term.getValue()))); + + return builder; + } + }, + not(TermType.not) { + @Override + public Query.Builder process(Term term, Query.Builder builder) { + builder.bool(bool -> bool + .mustNot(mustNot -> mustNot + .term(termQuery -> termQuery + .field(term.getColumn().trim()) + .value(FieldValue.of(term.getValue()))))); + return builder; + } + }, + isnull(TermType.isnull) { + @Override + public Query.Builder process(Term term, Query.Builder builder) { + builder + .bool(bool -> bool + .mustNot(mustNot -> notnull.process(term, mustNot))); + return builder; + } + + @Override + public Object convertTermValue(DataType type, Object value) { + return value; + } + }, + notnull(TermType.notnull) { + @Override + public Query.Builder process(Term term, Query.Builder builder) { + builder.exists(exists -> exists.field(term.getColumn())); + return builder; + } + + @Override + public Object convertTermValue(DataType type, Object value) { + return value; + } + }, + empty(TermType.empty) { + @Override + public Query.Builder process(Term term, Query.Builder builder) { + builder.term(t -> t.field(term.getColumn().trim()).value("")); + return builder; + } + + @Override + public Object convertTermValue(DataType type, Object value) { + return value; + } + }, + nempty(TermType.nempty) { + @Override + public Query.Builder process(Term term, Query.Builder builder) { + builder.bool(bool -> bool + .mustNot(mustNot -> empty.process(term, mustNot))); + return builder; + } + + @Override + public Object convertTermValue(DataType type, Object value) { + return value; + } + }, + btw(TermType.btw) { + @Override + public Query.Builder process(Term term, Query.Builder builder) { + Number between = null; + Number and = null; + List values = ConverterUtils.convertToList(term.getValue()); + int size = values.size(); + if (size > 0) { + between = CastUtils.castNumber(values.get(0)); + } + if (size > 1) { + and = CastUtils.castNumber(values.get(1)); + } + Number fb = between, ab = and; + builder.range(range -> range + .untyped(numberRange -> { + numberRange.gte(JsonData.of(fb)); + numberRange.lte(JsonData.of(ab)); + return numberRange.field(term.getColumn()); + })); + return builder; + } + }, + gt(TermType.gt) { + @Override + public Query.Builder process(Term term, Query.Builder builder) { + Number value = CastUtils.castNumber(term.getValue()); + builder.range(range -> range + .untyped(untyped -> untyped + .field(term.getColumn()) + .gt(JsonData.of(value)))); + return builder; + } + }, + gte(TermType.gte) { + @Override + public Query.Builder process(Term term, Query.Builder builder) { + Number value = CastUtils.castNumber(term.getValue()); + builder.range(range -> range + .untyped(untyped -> untyped + .field(term.getColumn()) + .gte(JsonData.of(value)))); + return builder; + } + }, + lt(TermType.lt) { + @Override + public Query.Builder process(Term term, Query.Builder builder) { + Number value = CastUtils.castNumber(term.getValue()); + builder.range(range -> range + .untyped(untyped -> untyped + .field(term.getColumn()) + .lt(JsonData.of(value)))); + return builder; + } + }, + lte(TermType.lte) { + @Override + public Query.Builder process(Term term, Query.Builder builder) { + Number value = CastUtils.castNumber(term.getValue()); + builder.range(range -> range + .untyped(untyped -> untyped + .field(term.getColumn()) + .lte(JsonData.of(value)))); + return builder; + } + }, + in(TermType.in) { + @Override + public Query.Builder process(Term term, Query.Builder builder) { + builder.terms(termQuery -> termQuery + .field(term.getColumn().trim()) + .terms(t -> t.value( + ConverterUtils.convertToList(term.getValue(), FieldValue::of) + ))); + return builder; + } + }, + nin(TermType.nin) { + @Override + public Query.Builder process(Term term, Query.Builder builder) { + builder.bool(bool -> bool + .mustNot(mustNot -> in.process(term, mustNot))); + return builder; + } + }, + like(TermType.like) { + @Override + public Query.Builder process(Term term, Query.Builder builder) { + builder.wildcard(wildcard -> wildcard + .field(term.getColumn().trim()) + .value(likeQueryTermValueHandler(term.getValue()))); + return builder; + } + }, + nlike(TermType.nlike) { + @Override + public Query.Builder process(Term term, Query.Builder builder) { + builder.bool(bool -> bool + .mustNot(mustNot -> mustNot + .wildcard(wildcard -> wildcard + .field(term.getColumn().trim()) + .value(likeQueryTermValueHandler(term.getValue()))))); + return builder; + } + }; + + private final String type; + + @Override + public boolean isSupported(Term term) { + return this.type.equalsIgnoreCase(term.getTermType()); + } + + @Override + public String getId() { + return name(); + } + + private static String convertString(Object value) { + if (value instanceof Collection) { + return String.join(",", ((Collection) value)); + } else { + return String.valueOf(value); + } + } + + public static String likeQueryTermValueHandler(Object value) { + if (!ObjectUtils.isEmpty(value)) { + return convertString(value).replace("%", "*"); + } + return "**"; + } + + public static Optional of(String type) { + return Arrays.stream(values()) + .filter(e -> e.getType().equalsIgnoreCase(type)) + .findAny(); + } +} diff --git a/jetlinks-components/elasticsearch-component/elasticsearch-8x/src/main/resources/META-INF/services/org.jetlinks.community.elastic.search.ElasticSearchSupport b/jetlinks-components/elasticsearch-component/elasticsearch-8x/src/main/resources/META-INF/services/org.jetlinks.community.elastic.search.ElasticSearchSupport new file mode 100644 index 00000000..40dbbb57 --- /dev/null +++ b/jetlinks-components/elasticsearch-component/elasticsearch-8x/src/main/resources/META-INF/services/org.jetlinks.community.elastic.search.ElasticSearchSupport @@ -0,0 +1 @@ +org.jetlinks.community.elastic.search.ElasticSearch8xSupport \ No newline at end of file diff --git a/jetlinks-components/elasticsearch-component/elasticsearch-core/pom.xml b/jetlinks-components/elasticsearch-component/elasticsearch-core/pom.xml new file mode 100644 index 00000000..472cd169 --- /dev/null +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/pom.xml @@ -0,0 +1,129 @@ + + + 4.0.0 + + org.jetlinks.community + elasticsearch-component + 2.10.0-SNAPSHOT + ../pom.xml + + + elasticsearch-core + + + + + + + + + + + + + + + + + org.apache.logging.log4j + log4j-core + + + + org.slf4j + log4j-over-slf4j + + + + org.locationtech.jts + jts-core + 1.18.2 + + + + + org.locationtech.spatial4j + spatial4j + 0.8 + + + + + + + + + + + + + + org.hswebframework.web + hsweb-commons-crud + ${hsweb.framework.version} + + + + org.hswebframework + hsweb-easy-orm-rdb + + + + org.jetlinks + jetlinks-core + ${jetlinks.version} + + + + ${project.groupId} + timeseries-component + ${project.version} + + + + io.projectreactor.netty + reactor-netty + + + + org.jetlinks.community + configure-component + ${project.version} + + + + org.jetlinks.community + things-component + ${project.version} + true + + + + org.springframework.data + spring-data-elasticsearch + + + co.elastic.clients + elasticsearch-java + + + org.elasticsearch.client + elasticsearch-rest-client + + + + + + co.elastic.clients + elasticsearch-java + + + org.elasticsearch.client + elasticsearch-rest-client + + + + + \ No newline at end of file diff --git a/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/ElasticSearchSupport.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/ElasticSearchSupport.java new file mode 100644 index 00000000..6a32342a --- /dev/null +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/ElasticSearchSupport.java @@ -0,0 +1,111 @@ +package org.jetlinks.community.elastic.search; + +import co.elastic.clients.elasticsearch._types.aggregations.DateHistogramBucket; +import co.elastic.clients.elasticsearch._types.aggregations.HistogramBucket; +import co.elastic.clients.elasticsearch._types.aggregations.MultiBucketBase; +import co.elastic.clients.elasticsearch._types.mapping.DynamicTemplate; +import co.elastic.clients.elasticsearch._types.mapping.Property; +import co.elastic.clients.elasticsearch.indices.*; +import co.elastic.clients.elasticsearch.indices.get_mapping.IndexMappingRecord; +import co.elastic.clients.json.JsonData; +import co.elastic.clients.transport.Version; +import com.google.common.collect.Maps; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.MapUtils; +import org.jetlinks.community.elastic.search.index.ElasticSearchIndexProperties; + +import java.util.HashMap; +import java.util.Map; +import java.util.ServiceLoader; + +@Slf4j +public abstract class ElasticSearchSupport { + + private static ElasticSearchSupport support; + + static { + ServiceLoader + .load(ElasticSearchSupport.class) + .forEach(ElasticSearchSupport::register); + } + + public static void load() { + + } + + public int majorVersion() { + return Version.VERSION == null ? 8 : Version.VERSION.major(); + } + + public boolean is8x() { + return majorVersion() == 8; + } + + public boolean is7x() { + return majorVersion() == 7; + } + + public static ElasticSearchSupport current() { + if (support == null) { + throw new UnsupportedOperationException("当前环境不支持elasticsearch,请添加依赖`elasticsearch-8x`."); + } + return support; + } + + private static void register(ElasticSearchSupport support) { + if (ElasticSearchSupport.support != null) { + log.warn("ignore register elasticsearch support:{}", support.getClass()); + return; + } + support.setup(); + + ElasticSearchSupport.support = support; + + log.info("register elasticsearch support:{}", support.getClass()); + } + + + protected void setup() { + + } + + public IndexSettings.Builder applyIndexSettings(ElasticSearchIndexProperties index, + IndexSettings.Builder builder) { + + builder.numberOfShards(String.valueOf(Math.max(1, index.getNumberOfShards()))) + .numberOfReplicas(String.valueOf(index.getNumberOfReplicas())); + + if (MapUtils.isNotEmpty(index.getOptions())) { + builder.otherSettings(Maps.transformValues(index.getOptions(), JsonData::of)); + } + return builder; + } + + public abstract DynamicTemplate createDynamicTemplate(String type, Property property); + + public abstract IndexState getIndexState(GetIndexResponse response, String index); + + public abstract TemplateMapping getTemplateMapping(GetTemplateResponse response, String index); + + public abstract IndexMappingRecord getIndexMapping(GetMappingResponse response, String index); + + + protected abstract Object getBucketKey(MultiBucketBase bucket); + + public Map transformBucket(String name, + Map map, + MultiBucketBase bucket) { + if (bucket instanceof DateHistogramBucket _bucket) { + Map val = new HashMap<>(map); + val.put(name, _bucket.keyAsString()); + val.put("_" + name, getBucketKey(_bucket)); + return val; + } else if (bucket instanceof HistogramBucket _bucket) { + Map val = new HashMap<>(map); + val.put(name, _bucket.keyAsString()); + val.put("_" + name, getBucketKey(_bucket)); + return val; + } + return map; + } +} diff --git a/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/configuration/ElasticSearchClientConfiguration.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/configuration/ElasticSearchClientConfiguration.java new file mode 100755 index 00000000..916d13b1 --- /dev/null +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/configuration/ElasticSearchClientConfiguration.java @@ -0,0 +1,44 @@ +package org.jetlinks.community.elastic.search.configuration; + +import co.elastic.clients.json.JsonpMapper; +import co.elastic.clients.json.jackson.JacksonJsonpMapper; +import co.elastic.clients.transport.rest_client.RestClientOptions; +import co.elastic.clients.transport.rest_client.RestClientTransport; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; +import org.elasticsearch.client.RestClient; +import org.jetlinks.community.elastic.search.trace.TraceInstrumentation; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.beans.factory.config.BeanDefinition; +import org.springframework.boot.autoconfigure.AutoConfiguration; +import org.springframework.boot.autoconfigure.elasticsearch.ReactiveElasticsearchClientAutoConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Role; + +/** + * @author zhouhao + * @since 2.10 + **/ +@AutoConfiguration(before = ReactiveElasticsearchClientAutoConfiguration.class) +@Slf4j +@Role(BeanDefinition.ROLE_INFRASTRUCTURE) +public class ElasticSearchClientConfiguration { + + @Bean + JacksonJsonpMapper jacksonJsonpMapper(ObjectMapper mapper) { + return new JacksonJsonpMapper(mapper); + } + + @Bean + RestClientTransport restClientTransport(RestClient restClient, JsonpMapper jsonMapper, + ObjectProvider restClientOptions) { + return new RestClientTransport( + restClient, + jsonMapper, + restClientOptions.getIfAvailable(), + new TraceInstrumentation()); + } + + + +} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/configuration/ElasticSearchConfiguration.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/configuration/ElasticSearchConfiguration.java similarity index 53% rename from jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/configuration/ElasticSearchConfiguration.java rename to jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/configuration/ElasticSearchConfiguration.java index da55377d..d0c8cf19 100755 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/configuration/ElasticSearchConfiguration.java +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/configuration/ElasticSearchConfiguration.java @@ -1,35 +1,38 @@ package org.jetlinks.community.elastic.search.configuration; +import co.elastic.clients.elasticsearch.ElasticsearchClient; +import co.elastic.clients.json.JsonpMapper; +import co.elastic.clients.json.jackson.JacksonJsonpMapper; +import co.elastic.clients.transport.rest_client.RestClientOptions; +import co.elastic.clients.transport.rest_client.RestClientTransport; +import com.fasterxml.jackson.databind.ObjectMapper; import lombok.Generated; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; -import org.jetlinks.community.elastic.search.embedded.EmbeddedElasticSearch; -import org.jetlinks.community.elastic.search.embedded.EmbeddedElasticSearchProperties; -import org.jetlinks.community.elastic.search.index.DefaultElasticSearchIndexManager; -import org.jetlinks.community.elastic.search.index.ElasticSearchIndexManager; -import org.jetlinks.community.elastic.search.index.ElasticSearchIndexProperties; -import org.jetlinks.community.elastic.search.index.ElasticSearchIndexStrategy; -import org.jetlinks.community.elastic.search.index.strategies.DirectElasticSearchIndexStrategy; -import org.jetlinks.community.elastic.search.index.strategies.TimeByMonthElasticSearchIndexStrategy; +import org.elasticsearch.client.RestClient; +import org.jetlinks.community.elastic.search.index.*; +import org.jetlinks.community.elastic.search.index.strategies.*; +import org.jetlinks.community.elastic.search.service.reactive.*; +import org.jetlinks.community.elastic.search.index.*; +import org.jetlinks.community.elastic.search.index.strategies.*; import org.jetlinks.community.elastic.search.service.AggregationService; import org.jetlinks.community.elastic.search.service.ElasticSearchService; import org.jetlinks.community.elastic.search.service.reactive.*; import org.jetlinks.community.elastic.search.timeseries.ElasticSearchTimeSeriesManager; +import org.springframework.beans.factory.ObjectProvider; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.AutoConfigureAfter; -import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.beans.factory.config.BeanDefinition; +import org.springframework.boot.autoconfigure.AutoConfiguration; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; -import org.springframework.boot.autoconfigure.data.elasticsearch.ReactiveElasticsearchRestClientAutoConfiguration; +import org.springframework.boot.autoconfigure.elasticsearch.ReactiveElasticsearchClientAutoConfiguration; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.DependsOn; import org.springframework.context.annotation.Primary; -import org.springframework.data.elasticsearch.client.ClientConfiguration; -import org.springframework.data.elasticsearch.client.reactive.HostProvider; -import org.springframework.data.elasticsearch.client.reactive.RequestCreator; -import org.springframework.data.elasticsearch.client.reactive.WebClientProvider; +import org.springframework.context.annotation.Role; +import org.springframework.core.Ordered; +import org.springframework.core.annotation.Order; -import java.net.InetSocketAddress; import java.util.List; /** @@ -37,50 +40,24 @@ import java.util.List; * @author zhouhao * @since 1.0 **/ -@Configuration(proxyBeanMethods = false) +@AutoConfiguration(after = ReactiveElasticsearchClientAutoConfiguration.class) @Slf4j @EnableConfigurationProperties({ - EmbeddedElasticSearchProperties.class, ElasticSearchIndexProperties.class, ElasticSearchBufferProperties.class}) -@AutoConfigureAfter(ReactiveElasticsearchRestClientAutoConfiguration.class) -@ConditionalOnBean(ClientConfiguration.class) @Generated +@Role(BeanDefinition.ROLE_INFRASTRUCTURE) public class ElasticSearchConfiguration { + @Bean @SneakyThrows @Primary - public DefaultReactiveElasticsearchClient defaultReactiveElasticsearchClient(EmbeddedElasticSearchProperties embeddedProperties, - ClientConfiguration clientConfiguration) { - if (embeddedProperties.isEnabled()) { - log.debug("starting embedded elasticsearch on {}:{}", - embeddedProperties.getHost(), - embeddedProperties.getPort()); - - new EmbeddedElasticSearch(embeddedProperties).start(); - } - WebClientProvider provider = getWebClientProvider(clientConfiguration); - - HostProvider hostProvider = HostProvider.provider(provider, - clientConfiguration.getHeadersSupplier(), - clientConfiguration - .getEndpoints() - .toArray(new InetSocketAddress[0])); - - DefaultReactiveElasticsearchClient client = - new DefaultReactiveElasticsearchClient(hostProvider, new RequestCreator() { - }); - - client.setHeadersSupplier(clientConfiguration.getHeadersSupplier()); - - return client; + @Role(BeanDefinition.ROLE_INFRASTRUCTURE) + public DefaultReactiveElasticsearchClient defaultReactiveElasticsearchClient(ElasticsearchClient client) { + return new DefaultReactiveElasticsearchClient(client); } - private static WebClientProvider getWebClientProvider(ClientConfiguration clientConfiguration) { - - return WebClientProvider.getWebClientProvider(clientConfiguration); - } @Bean public DirectElasticSearchIndexStrategy directElasticSearchIndexStrategy(ReactiveElasticsearchClient elasticsearchClient, @@ -89,18 +66,40 @@ public class ElasticSearchConfiguration { } @Bean + @Role(BeanDefinition.ROLE_INFRASTRUCTURE) public TimeByMonthElasticSearchIndexStrategy timeByMonthElasticSearchIndexStrategy(ReactiveElasticsearchClient elasticsearchClient, ElasticSearchIndexProperties indexProperties) { return new TimeByMonthElasticSearchIndexStrategy(elasticsearchClient, indexProperties); } @Bean - public DefaultElasticSearchIndexManager elasticSearchIndexManager(@Autowired(required = false) List strategies) { - return new DefaultElasticSearchIndexManager(strategies); + public TimeByDayElasticSearchIndexStrategy timeByDayElasticSearchIndexStrategy(ReactiveElasticsearchClient elasticsearchClient, + ElasticSearchIndexProperties indexProperties) { + return new TimeByDayElasticSearchIndexStrategy(elasticsearchClient, indexProperties); } @Bean + public TimeByWeekElasticSearchIndexStrategy timeByWeekElasticSearchIndexStrategy(ReactiveElasticsearchClient elasticsearchClient, + ElasticSearchIndexProperties indexProperties){ + return new TimeByWeekElasticSearchIndexStrategy(elasticsearchClient, indexProperties); + } + + @Bean + public AffixesElasticSearchIndexStrategy affixesElasticSearchIndexStrategy(ReactiveElasticsearchClient elasticsearchClient, + ElasticSearchIndexProperties indexProperties) { + return new AffixesElasticSearchIndexStrategy(elasticsearchClient, indexProperties); + } + + @Bean + public DefaultElasticSearchIndexManager elasticSearchIndexManager(@Autowired(required = false) List strategies, + @Autowired(required = false) List customizers) { + return new DefaultElasticSearchIndexManager(strategies, customizers); + } + + @Order(Ordered.HIGHEST_PRECEDENCE + 1000) + @Bean(destroyMethod = "shutdown") @ConditionalOnMissingBean(ElasticSearchService.class) + @DependsOn("defaultReactiveElasticsearchClient") public ReactiveElasticSearchService reactiveElasticSearchService(ReactiveElasticsearchClient elasticsearchClient, ElasticSearchIndexManager indexManager, ElasticSearchBufferProperties properties) { diff --git a/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/configuration/ElasticSearchProperties.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/configuration/ElasticSearchProperties.java new file mode 100644 index 00000000..d13e0f5f --- /dev/null +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/configuration/ElasticSearchProperties.java @@ -0,0 +1,6 @@ +package org.jetlinks.community.elastic.search.configuration; + +public class ElasticSearchProperties { + + +} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/configuration/ElasticSearchThingDataConfiguration.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/configuration/ElasticSearchThingDataConfiguration.java similarity index 90% rename from jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/configuration/ElasticSearchThingDataConfiguration.java rename to jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/configuration/ElasticSearchThingDataConfiguration.java index 12b77574..4807c0b1 100644 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/configuration/ElasticSearchThingDataConfiguration.java +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/configuration/ElasticSearchThingDataConfiguration.java @@ -7,6 +7,7 @@ import org.jetlinks.community.elastic.search.service.ElasticSearchService; import org.jetlinks.community.elastic.search.things.ElasticSearchColumnModeStrategy; import org.jetlinks.community.elastic.search.things.ElasticSearchRowModeStrategy; import org.jetlinks.community.things.data.ThingsDataRepositoryStrategy; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -16,6 +17,7 @@ import org.springframework.context.annotation.Configuration; public class ElasticSearchThingDataConfiguration { @Bean + @ConditionalOnBean(ElasticSearchService.class) public ElasticSearchColumnModeStrategy elasticSearchColumnModThingDataPolicy( ThingsRegistry registry, ElasticSearchService searchService, @@ -26,6 +28,7 @@ public class ElasticSearchThingDataConfiguration { } @Bean + @ConditionalOnBean(ElasticSearchService.class) public ElasticSearchRowModeStrategy elasticSearchRowModThingDataPolicy( ThingsRegistry registry, ElasticSearchService searchService, diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticDateFormat.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticDateFormat.java old mode 100644 new mode 100755 similarity index 84% rename from jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticDateFormat.java rename to jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticDateFormat.java index cdedbf75..8538442f --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticDateFormat.java +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticDateFormat.java @@ -9,9 +9,10 @@ import java.util.List; import java.util.stream.Collectors; /** + * Values based on reference doc - https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-date-format.html + * * @author bsetfeng * @since 1.0 - * Values based on reference doc - https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-date-format.html **/ @Getter @AllArgsConstructor @@ -24,11 +25,11 @@ public enum ElasticDateFormat implements EnumDict { strict_date_time("strict_date_time", "yyyy-MM-dd'T'HH:mm:ss.SSSZZ"), strict_date_hour_minute_second("strict_date_hour_minute_second", "yyyy-MM-dd'T'HH:mm:ss"), strict_hour_minute_second("strict_hour_minute_second", "HH:mm:ss"), - simple_date("yyyy-MM-dd HH:mm:ss", "通用格式"); + simple_date("8yyyy-MM-dd HH:mm:ss", "通用格式"); private String value; - private final String text; + private String text; public static String getFormat(ElasticDateFormat... dateFormats) { return getFormat(Arrays.asList(dateFormats)); @@ -36,13 +37,13 @@ public enum ElasticDateFormat implements EnumDict { public static String getFormat(List dateFormats) { return getFormatStr(dateFormats.stream() - .map(ElasticDateFormat::getValue) - .collect(Collectors.toList()) + .map(ElasticDateFormat::getValue) + .collect(Collectors.toList()) ); } public static String getFormatStr(List dateFormats) { - StringBuffer format = new StringBuffer(); + StringBuilder format = new StringBuilder(); for (int i = 0; i < dateFormats.size(); i++) { format.append(dateFormats.get(i)); if (i != dateFormats.size() - 1) { diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticPropertyType.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticPropertyType.java old mode 100644 new mode 100755 similarity index 74% rename from jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticPropertyType.java rename to jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticPropertyType.java index a1257729..58712896 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticPropertyType.java +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticPropertyType.java @@ -29,14 +29,17 @@ public enum ElasticPropertyType implements EnumDict { IP("ip", "ip", LongType::new), ATTACHMENT("attachment", "attachment", FileType::new), KEYWORD("string", "keyword", StringType::new), - GEO_POINT("geo_point", "geo_point", GeoType::new); + GEO_POINT("geo_point", "geo_point", GeoType::new), + GEO_SHAPE("geo_shape", "geo_shape", GeoShapeType::new) + + ; @Getter - private String text; + private final String text; @Getter - private String value; + private final String value; - private Supplier typeBuilder; + private final Supplier typeBuilder; public DataType getType() { return typeBuilder.get(); @@ -53,14 +56,4 @@ public enum ElasticPropertyType implements EnumDict { return null; } - public static ElasticPropertyType ofJava(Object value) { - if (!StringUtils.isEmpty(value)) { - for (ElasticPropertyType elasticPropertyType : ElasticPropertyType.values()) { - if (elasticPropertyType.getText().equals(value)) { - return elasticPropertyType; - } - } - } - throw new NotFoundException("未找到数据类型为：" + value + "的枚举"); - } } diff --git a/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticSearchTermType.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticSearchTermType.java new file mode 100644 index 00000000..f8c5acf6 --- /dev/null +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticSearchTermType.java @@ -0,0 +1,26 @@ +package org.jetlinks.community.elastic.search.enums; + +import co.elastic.clients.elasticsearch._types.query_dsl.Query; +import org.hswebframework.ezorm.core.param.Term; +import org.jetlinks.core.metadata.DataType; +import org.jetlinks.community.spi.Provider; +import org.jetlinks.community.things.utils.ThingsDatabaseUtils; + +public interface ElasticSearchTermType { + + Provider supports = Provider.create(ElasticSearchTermType.class); + + String getId(); + + boolean isSupported(Term term); + + Query.Builder process(Term term, Query.Builder builder); + + default Query process(Term term) { + return process(term, new Query.Builder()).build(); + } + + default Object convertTermValue(DataType type, Object value) { + return ThingsDatabaseUtils.tryConvertTermValue(type, value); + } +} diff --git a/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticSearchTermTypes.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticSearchTermTypes.java new file mode 100644 index 00000000..73e6418c --- /dev/null +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/enums/ElasticSearchTermTypes.java @@ -0,0 +1,41 @@ +package org.jetlinks.community.elastic.search.enums; + +import org.hswebframework.ezorm.core.param.Term; + +import java.util.Optional; + +public class ElasticSearchTermTypes { + + + static { + + } + + public static void register(ElasticSearchTermType termType) { + ElasticSearchTermType.supports.register(termType.getId(), termType); + } + + public static ElasticSearchTermType lookupNow(Term term) { + + return lookup(term) + .orElseThrow(() -> new UnsupportedOperationException("不支持的查询条件:" + term.getType())); + } + + public static Optional lookup(Term term) { + ElasticSearchTermType fast = ElasticSearchTermType + .supports + .get(term.getTermType()) + .orElse(null); + if (fast != null) { + return Optional.of(fast); + } + + for (ElasticSearchTermType termType : ElasticSearchTermType.supports.getAll()) { + if (termType.isSupported(term)) { + return Optional.of(termType); + } + } + return Optional.empty(); + } + +} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/DefaultElasticSearchIndexManager.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/DefaultElasticSearchIndexManager.java similarity index 79% rename from jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/DefaultElasticSearchIndexManager.java rename to jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/DefaultElasticSearchIndexManager.java index f353af97..9dad582e 100755 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/DefaultElasticSearchIndexManager.java +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/DefaultElasticSearchIndexManager.java @@ -3,9 +3,12 @@ package org.jetlinks.community.elastic.search.index; import lombok.Generated; import lombok.Getter; import lombok.Setter; +import org.apache.commons.collections4.map.CaseInsensitiveMap; +import org.jetlinks.core.cache.Caches; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.stereotype.Component; +import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import java.util.List; @@ -20,6 +23,12 @@ public class DefaultElasticSearchIndexManager implements ElasticSearchIndexManag @Generated private String defaultStrategy = "direct"; + @Getter + @Setter + @Generated + //是否自动创建索引 + private boolean autoCreate = true; + @Getter @Setter @Generated @@ -29,14 +38,23 @@ public class DefaultElasticSearchIndexManager implements ElasticSearchIndexManag private final Map indexMetadataStore = new ConcurrentSkipListMap<>(String.CASE_INSENSITIVE_ORDER); - public DefaultElasticSearchIndexManager(@Autowired(required = false) List strategies) { + public DefaultElasticSearchIndexManager(@Autowired(required = false) List strategies, + @Autowired(required = false) List customizers) { if (strategies != null) { strategies.forEach(this::registerStrategy); } + if (customizers != null) { + customizers.forEach(customizer -> customizer.custom(this)); + } } @Override public Mono putIndex(ElasticSearchIndexMetadata index) { + //禁用索引创建 + if (!autoCreate) { + indexMetadataStore.put(index.getIndex(), index); + return Mono.empty(); + } return this.getIndexStrategy(index.getIndex()) .flatMap(strategy -> strategy.putIndex(index)) .doOnNext(idx -> indexMetadataStore.put(idx.getIndex(), idx)) diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/DefaultElasticSearchIndexMetadata.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/DefaultElasticSearchIndexMetadata.java old mode 100644 new mode 100755 similarity index 82% rename from jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/DefaultElasticSearchIndexMetadata.java rename to jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/DefaultElasticSearchIndexMetadata.java index 004d3dee..82577582 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/DefaultElasticSearchIndexMetadata.java +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/DefaultElasticSearchIndexMetadata.java @@ -1,5 +1,6 @@ package org.jetlinks.community.elastic.search.index; +import lombok.Generated; import org.jetlinks.core.metadata.DataType; import org.jetlinks.core.metadata.PropertyMetadata; import org.jetlinks.core.metadata.SimplePropertyMetadata; @@ -10,9 +11,9 @@ import java.util.List; import java.util.Map; public class DefaultElasticSearchIndexMetadata implements ElasticSearchIndexMetadata { - private String index; + private final String index; - private Map properties = new HashMap<>(); + private final Map properties = new HashMap<>(); public DefaultElasticSearchIndexMetadata(String index) { this.index = index.toLowerCase().trim(); @@ -24,16 +25,19 @@ public class DefaultElasticSearchIndexMetadata implements ElasticSearchIndexMeta } @Override + @Generated public PropertyMetadata getProperty(String property) { return properties.get(property); } @Override + @Generated public String getIndex() { return index; } @Override + @Generated public List getProperties() { return new ArrayList<>(properties.values()); } @@ -44,10 +48,10 @@ public class DefaultElasticSearchIndexMetadata implements ElasticSearchIndexMeta } public DefaultElasticSearchIndexMetadata addProperty(String property, DataType type) { - SimplePropertyMetadata metadata=new SimplePropertyMetadata(); + SimplePropertyMetadata metadata = new SimplePropertyMetadata(); metadata.setValueType(type); metadata.setId(property); - properties.put(property, metadata); + addProperty(metadata); return this; } } diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/ElasticIndex.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/ElasticIndex.java old mode 100644 new mode 100755 similarity index 100% rename from jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/ElasticIndex.java rename to jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/ElasticIndex.java diff --git a/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexCustomizer.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexCustomizer.java new file mode 100755 index 00000000..f3be80eb --- /dev/null +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexCustomizer.java @@ -0,0 +1,7 @@ +package org.jetlinks.community.elastic.search.index; + +public interface ElasticSearchIndexCustomizer { + + void custom(ElasticSearchIndexManager manager); + +} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexManager.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexManager.java old mode 100644 new mode 100755 similarity index 85% rename from jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexManager.java rename to jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexManager.java index d26e406f..d1dbbd5d --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexManager.java +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexManager.java @@ -3,6 +3,12 @@ package org.jetlinks.community.elastic.search.index; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; +/** + * ElasticSearch 索引管理器,用于统一管理,维护索引信息. + * + * @author zhouhao + * @since 1.0 + */ public interface ElasticSearchIndexManager { /** @@ -23,6 +29,7 @@ public interface ElasticSearchIndexManager { /** * 获取多个所有元数据 + * * @param index 索引名称 * @return 索引元数据 */ @@ -41,7 +48,13 @@ public interface ElasticSearchIndexManager { */ Mono getIndexStrategy(String index); - default Flux getIndexesStrategy(String... index){ + /** + * 获取多个索引的策略 + * + * @param index 索引列表 + * @return 索引策略 + */ + default Flux getIndexesStrategy(String... index) { return Flux .fromArray(index) .flatMap(this::getIndexStrategy); diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexMetadata.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexMetadata.java old mode 100644 new mode 100755 similarity index 100% rename from jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexMetadata.java rename to jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexMetadata.java index 42546164..f9f424d2 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexMetadata.java +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexMetadata.java @@ -1,7 +1,7 @@ package org.jetlinks.community.elastic.search.index; -import org.jetlinks.community.elastic.search.utils.ElasticSearchConverter; import org.jetlinks.core.metadata.PropertyMetadata; +import org.jetlinks.community.elastic.search.utils.ElasticSearchConverter; import java.util.List; import java.util.Map; diff --git a/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexProperties.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexProperties.java new file mode 100755 index 00000000..7085fd1a --- /dev/null +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexProperties.java @@ -0,0 +1,58 @@ +package org.jetlinks.community.elastic.search.index; + +import co.elastic.clients.elasticsearch.indices.IndexSettings; +import co.elastic.clients.json.JsonData; +import com.google.common.collect.Maps; +import lombok.*; +import org.apache.commons.collections4.MapUtils; +import org.jetlinks.community.elastic.search.ElasticSearchSupport; +import org.springframework.boot.context.properties.ConfigurationProperties; + +import java.util.HashMap; +import java.util.Map; + +@Getter +@Setter +@Builder +@NoArgsConstructor +@AllArgsConstructor +@ConfigurationProperties(prefix = "elasticsearch.index.settings") +@Generated +public class ElasticSearchIndexProperties { + + //索引分片数据,通常为es的集群节点数量 + private int numberOfShards = 1; + + //副本数量 + private int numberOfReplicas = 0; + + //字段数量限制 + private long totalFieldsLimit = 2000; + + //默认字符串超过512将不会被索引,无法进行搜索 + private int keywordIgnoreAbove = 512; + + //其他的配置信息,在创建索引时将会设置到settings中 + private Map options; + + //是否使用别名进行搜索 + //设置为true将使用别名进行搜索,可通过手动绑定和接触别名来灵活配置搜索到的数据范围. + //设置为false时,将使用*进行搜索.在一些特殊请求,如索引名前缀类似时可能搜索到错误的数据. + private boolean useAliasSearch = true; + + + public IndexSettings.Builder toSettings(IndexSettings.Builder builder) { + + return ElasticSearchSupport + .current() + .applyIndexSettings(this, builder); + + } + + public void addSetting(String key, String value) { + if (null == options) { + options = new HashMap<>(); + } + options.put(key, value); + } +} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexStrategy.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexStrategy.java old mode 100644 new mode 100755 similarity index 89% rename from jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexStrategy.java rename to jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexStrategy.java index 0ee05c75..262c702a --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexStrategy.java +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexStrategy.java @@ -41,5 +41,10 @@ public interface ElasticSearchIndexStrategy { */ Mono putIndex(ElasticSearchIndexMetadata metadata); + /** + * 加载索引元数据 + * @param index 索引 + * @return 索引元数据 + */ Mono loadIndexMetadata(String index); } diff --git a/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/strategies/AbstractElasticSearchIndexStrategy.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/strategies/AbstractElasticSearchIndexStrategy.java new file mode 100755 index 00000000..54c1a274 --- /dev/null +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/strategies/AbstractElasticSearchIndexStrategy.java @@ -0,0 +1,277 @@ +package org.jetlinks.community.elastic.search.index.strategies; + +import co.elastic.clients.elasticsearch._types.mapping.DynamicTemplate; +import co.elastic.clients.elasticsearch._types.mapping.Property; +import co.elastic.clients.elasticsearch._types.mapping.TypeMapping; +import co.elastic.clients.elasticsearch.indices.PutMappingRequest; +import co.elastic.clients.elasticsearch.indices.get_mapping.IndexMappingRecord; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.jetlinks.core.metadata.DataType; +import org.jetlinks.core.metadata.PropertyMetadata; +import org.jetlinks.core.metadata.SimplePropertyMetadata; +import org.jetlinks.core.metadata.types.*; +import org.jetlinks.community.ConfigMetadataConstants; +import org.jetlinks.community.elastic.search.ElasticSearchSupport; +import org.jetlinks.community.elastic.search.enums.ElasticDateFormat; +import org.jetlinks.community.elastic.search.enums.ElasticPropertyType; +import org.jetlinks.community.elastic.search.index.DefaultElasticSearchIndexMetadata; +import org.jetlinks.community.elastic.search.index.ElasticSearchIndexMetadata; +import org.jetlinks.community.elastic.search.index.ElasticSearchIndexProperties; +import org.jetlinks.community.elastic.search.index.ElasticSearchIndexStrategy; +import org.jetlinks.community.elastic.search.service.reactive.ReactiveElasticsearchClient; +import org.jetlinks.reactor.ql.utils.CastUtils; +import org.springframework.util.CollectionUtils; +import reactor.core.publisher.Mono; + +import java.util.*; +import java.util.stream.Collectors; + +@AllArgsConstructor +@Slf4j +public abstract class AbstractElasticSearchIndexStrategy implements ElasticSearchIndexStrategy { + @Getter + private final String id; + + protected ReactiveElasticsearchClient client; + + protected ElasticSearchIndexProperties properties; + + protected String wrapIndex(String index) { + return index.toLowerCase(); + } + + protected Mono indexExists(String index) { + return client.execute( + c -> c + .indices() + .exists(b -> b.index(index)).value()); + } + + protected Mono doCreateIndex(ElasticSearchIndexMetadata metadata) { + return client + .execute(c -> { + c.indices() + .create(builder -> createIndexRequest(builder, metadata)); + return null; + }); + } + + protected Mono doPutIndex(ElasticSearchIndexMetadata metadata, + boolean justUpdateMapping) { + String index = wrapIndex(metadata.getIndex()); + return this.indexExists(index) + .flatMap(exists -> { + if (exists) { + return doLoadIndexMetadata(index) + .flatMap(oldMapping -> { + PutMappingRequest.Builder builder = + createPutMappingRequest(metadata, oldMapping, new PutMappingRequest.Builder()); + //无需更新 + if (builder == null) { + return Mono.empty(); + } + return client + .execute(c -> c.indices().putMapping(builder.build())); + }) + .then(); + } + if (justUpdateMapping) { + return Mono.empty(); + } + return doCreateIndex(metadata); + }); + } + + protected Mono doLoadIndexMetadata(String _index) { + String index = wrapIndex(_index); + return client + .execute(c -> { + IndexMappingRecord record = + ElasticSearchSupport + .current() + .getIndexMapping( + c.indices() + .getMapping(b -> b + .ignoreUnavailable(true) + .allowNoIndices(true) + .index(index)), + index + ); + if (record == null || record.mappings() == null) { + return null; + } + return convertMetadata(index, record.mappings()); + }); + } + + + protected co.elastic.clients.elasticsearch.indices.CreateIndexRequest.Builder + createIndexRequest(co.elastic.clients.elasticsearch.indices.CreateIndexRequest.Builder builder, + ElasticSearchIndexMetadata metadata) { + builder.index(wrapIndex(metadata.getIndex())); + + builder.settings(properties::toSettings); + + builder.mappings(b -> { + + b.properties(createElasticProperties(metadata.getProperties())); + + b.dynamicTemplates(createDynamicTemplates()); + + return b; + }); + + return builder; + } + + private co.elastic.clients.elasticsearch.indices.PutMappingRequest.Builder + createPutMappingRequest(ElasticSearchIndexMetadata metadata, + ElasticSearchIndexMetadata ignore, + co.elastic.clients.elasticsearch.indices.PutMappingRequest.Builder builder) { + Map properties = createElasticProperties(metadata.getProperties()); + Map ignoreProperties = createElasticProperties(ignore.getProperties()); + for (Map.Entry en : ignoreProperties.entrySet()) { + log.trace("ignore update index [{}] mapping property:{},{}", wrapIndex(metadata.getIndex()), en.getKey(), en + .getValue()); + properties.remove(en.getKey()); + } + if (properties.isEmpty()) { + log.debug("ignore update index [{}] mapping", wrapIndex(metadata.getIndex())); + return null; + } + List allProperties = new ArrayList<>(); + allProperties.addAll(metadata.getProperties()); + allProperties.addAll(ignore.getProperties()); + + builder.index(wrapIndex(metadata.getIndex())); + + builder.properties(createElasticProperties(allProperties)); + + return builder; + } + + protected Map createElasticProperties(List metadata) { + if (metadata == null) { + return new HashMap<>(); + } + return metadata + .stream() + .collect(Collectors.toMap(PropertyMetadata::getId, + prop -> this.createElasticProperty(prop.getValueType()), (a, v) -> a)); + } + + protected Property createElasticProperty(DataType type) { + + if (type instanceof DateTimeType) { + return Property.of(b -> b + .date(b2 -> b2 + .format( + ElasticDateFormat.getFormat( + ElasticDateFormat.epoch_millis, + ElasticDateFormat.strict_date_hour_minute_second, + ElasticDateFormat.strict_date_time, + ElasticDateFormat.strict_date) + ))); + + } else if (type instanceof DoubleType) { + + return Property.of(b -> b.double_(b2 -> b2.nullValue(null))); + + } else if (type instanceof LongType) { + return Property.of(b -> b.long_(b2 -> b2.nullValue(null))); + } else if (type instanceof IntType) { + return Property.of(b -> b.integer(b2 -> b2.nullValue(null))); + } else if (type instanceof FloatType) { + return Property.of(b -> b.float_(b2 -> b2.nullValue(null))); + } else if (type instanceof BooleanType) { + return Property.of(b -> b.boolean_(b2 -> b2.nullValue(null))); + } else if (type instanceof GeoType) { + return Property.of(b -> b.geoPoint(b2 -> b2)); + } else if (type instanceof GeoShapeType) { + return Property.of(b -> b.geoShape(b2 -> b2)); + } else if (type instanceof ArrayType) { + return createElasticProperty(((ArrayType) type).getElementType()); + } else if (type instanceof ObjectType objectType) { + if (!CollectionUtils.isEmpty(objectType.getProperties())) { + return Property.of(b -> b + .nested(b2 -> b2.properties(createElasticProperties(objectType.getProperties())))); + } + return Property.of(b -> b.nested(b2 -> b2)); + } else { + int above = Optional + .ofNullable(type) + .flatMap(_type -> _type.getExpand(ConfigMetadataConstants.maxLength.getKey())) + .filter(val -> val instanceof Number || StringUtils.isNumeric(String.valueOf(val))) + .map(CastUtils::castNumber) + .map(Number::intValue) + .orElse(properties.getKeywordIgnoreAbove()); + + return Property.of(b -> b.keyword(b2 -> b2.ignoreAbove(above))); + } + } + + protected ElasticSearchIndexMetadata convertMetadata(String index, TypeMapping mapping) { + Map properties = mapping.properties(); + + + return new DefaultElasticSearchIndexMetadata(index, convertProperties(properties)); + } + + @SuppressWarnings("all") + protected List convertProperties(Map properties) { + return properties + .entrySet() + .stream() + .map(entry -> convertProperty(entry.getKey(), entry.getValue())) + .collect(Collectors.toList()); + + } + + private PropertyMetadata convertProperty(String property, Property prop) { + SimplePropertyMetadata metadata = new SimplePropertyMetadata(); + metadata.setId(property); + metadata.setName(property); + ElasticPropertyType elasticPropertyType = ElasticPropertyType.of(prop._kind().jsonValue()); + if (null != elasticPropertyType) { + DataType dataType = elasticPropertyType.getType(); + if ((elasticPropertyType == ElasticPropertyType.OBJECT + || elasticPropertyType == ElasticPropertyType.NESTED) + && dataType instanceof ObjectType) { + ObjectType objectType = ((ObjectType) dataType); + objectType.setProperties(convertProperties(prop.nested().properties())); + } + metadata.setValueType(dataType); + } else { + metadata.setValueType(StringType.GLOBAL); + } + return metadata; + } + + + protected List> createDynamicTemplates() { + + List> list = new ArrayList<>(); + { + list.add(Collections.singletonMap( + "string_fields", ElasticSearchSupport + .current() + .createDynamicTemplate( + "string", + createElasticProperty(StringType.GLOBAL)))); + + } + { + list.add(Collections.singletonMap( + "date_fields", ElasticSearchSupport + .current() + .createDynamicTemplate( + "string", + createElasticProperty(StringType.GLOBAL)))); + } + + return list; + } +} diff --git a/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/strategies/AffixesElasticSearchIndexStrategy.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/strategies/AffixesElasticSearchIndexStrategy.java new file mode 100644 index 00000000..2aed636b --- /dev/null +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/strategies/AffixesElasticSearchIndexStrategy.java @@ -0,0 +1,86 @@ +package org.jetlinks.community.elastic.search.index.strategies; + +import lombok.Getter; +import lombok.Setter; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.jetlinks.community.elastic.search.index.ElasticSearchIndexMetadata; +import org.jetlinks.community.elastic.search.index.ElasticSearchIndexProperties; +import org.jetlinks.community.elastic.search.service.reactive.ReactiveElasticsearchClient; +import org.springframework.boot.context.properties.ConfigurationProperties; +import reactor.core.publisher.Mono; + +/** + * 前后缀索引策略支持. + * 
{@code
+ *
+ *   elasticsearch:
+ *      index:
+ *          default-strategy: affixes
+ *          affixes:
+ *            prefix: "" #前缀
+ *            suffix: "_test" # 后缀
+ *            auto-create: false # 是否创建索引
+ *
+ * }
+ * + * @author zhouhao + * @since 2.2 + */ +@Getter +@Setter +@Slf4j +@ConfigurationProperties(prefix = "elasticsearch.index.affixes") +public class AffixesElasticSearchIndexStrategy extends AbstractElasticSearchIndexStrategy { + //前缀 + private String prefix = ""; + //后缀 + private String suffix = ""; + //是否自动创建索引 + private boolean autoCreate = true; + + public AffixesElasticSearchIndexStrategy(ReactiveElasticsearchClient client, + ElasticSearchIndexProperties properties) { + super("affixes", client, properties); + } + + + @Override + public String getIndexForSave(String index) { + return prefix + index + suffix; + } + + @Override + public String getIndexForSearch(String index) { + return prefix + index + suffix; + } + + @Override + @SneakyThrows + public Mono putIndex(ElasticSearchIndexMetadata metadata) { + + if (log.isInfoEnabled() && !autoCreate) { +// CreateIndexRequest request = createIndexRequest(metadata); +// Object data = ObjectMappers +// .parseJson( +// new RequestCreator() { +// } +// .createIndexRequest().apply(request).getEntity() +// .getContent(), Object.class); +// log.info("ignore put elasticsearch index [{}] :\n{}", metadata.getIndex(), JSON.toJSONString(data, SerializerFeature.PrettyFormat)); + } + + if (autoCreate) { + return this + .doPutIndex(metadata.newIndexName(getIndexForSave(metadata.getIndex())), false) + .thenReturn(metadata); + } + + return Mono.just(metadata); + } + + @Override + public Mono loadIndexMetadata(String index) { + return doLoadIndexMetadata(getIndexForSearch(index)); + } +} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/strategies/DirectElasticSearchIndexStrategy.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/strategies/DirectElasticSearchIndexStrategy.java similarity index 100% rename from jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/strategies/DirectElasticSearchIndexStrategy.java rename to jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/strategies/DirectElasticSearchIndexStrategy.java diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/strategies/TemplateElasticSearchIndexStrategy.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/strategies/TemplateElasticSearchIndexStrategy.java similarity index 52% rename from jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/strategies/TemplateElasticSearchIndexStrategy.java rename to jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/strategies/TemplateElasticSearchIndexStrategy.java index 8195aafb..8b8732fa 100755 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/strategies/TemplateElasticSearchIndexStrategy.java +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/strategies/TemplateElasticSearchIndexStrategy.java @@ -1,20 +1,16 @@ package org.jetlinks.community.elastic.search.index.strategies; +import co.elastic.clients.elasticsearch.indices.PutIndexTemplateRequest; +import co.elastic.clients.elasticsearch.indices.TemplateMapping; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; -import org.elasticsearch.Version; -import org.elasticsearch.action.admin.indices.alias.Alias; -import org.elasticsearch.client.indices.GetIndexTemplatesRequest; -import org.elasticsearch.client.indices.PutIndexTemplateRequest; +import org.jetlinks.community.elastic.search.ElasticSearchSupport; import org.jetlinks.community.elastic.search.index.ElasticSearchIndexMetadata; import org.jetlinks.community.elastic.search.index.ElasticSearchIndexProperties; import org.jetlinks.community.elastic.search.service.reactive.ReactiveElasticsearchClient; import reactor.core.publisher.Mono; import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; @Slf4j public abstract class TemplateElasticSearchIndexStrategy extends AbstractElasticSearchIndexStrategy { @@ -40,15 +36,21 @@ public abstract class TemplateElasticSearchIndexStrategy extends AbstractElastic @Override public String getIndexForSearch(String index) { - return getAlias(index); + if (properties.isUseAliasSearch()) { + return getAlias(index); + } else { + return wrapIndex(index).concat("*"); + } } @Override public Mono putIndex(ElasticSearchIndexMetadata metadata) { String saveIndex = getIndexForSave(metadata.getIndex()); + return client - .putTemplate(createIndexTemplateRequest(metadata)) - //修改当前索引 + .execute(c -> c + .indices() + .putIndexTemplate(request -> createIndexTemplateRequest(request, metadata))) .then(doPutIndex(metadata.newIndexName(saveIndex), true) //忽略修改索引错误 .onErrorResume(err -> { @@ -58,32 +60,44 @@ public abstract class TemplateElasticSearchIndexStrategy extends AbstractElastic .thenReturn(metadata.newIndexName(wrapIndex(metadata.getIndex()))); } - protected PutIndexTemplateRequest createIndexTemplateRequest(ElasticSearchIndexMetadata metadata) { + protected PutIndexTemplateRequest.Builder createIndexTemplateRequest(PutIndexTemplateRequest.Builder builder, + ElasticSearchIndexMetadata metadata) { String index = wrapIndex(metadata.getIndex()); - PutIndexTemplateRequest request = new PutIndexTemplateRequest(getTemplate(index)); - request.alias(new Alias(getAlias(index))); - request.settings(properties.toSettings()); - Map mappingConfig = new HashMap<>(); - mappingConfig.put("properties", createElasticProperties(metadata.getProperties())); - mappingConfig.put("dynamic_templates", createDynamicTemplates()); - mappingConfig.put("_source", Collections.singletonMap("enabled", true)); - if (client.serverVersion().after(Version.V_7_0_0)) { - request.mapping(mappingConfig); - } else { - request.mapping(Collections.singletonMap("_doc", mappingConfig)); + builder.name(getTemplate(index)); + + builder.indexPatterns(getIndexPatterns(index)); + + // 7.x不支持此设置 + if (ElasticSearchSupport.current().is8x()) { + builder.allowAutoCreate(true); } - request.patterns(getIndexPatterns(index)); - return request; + + builder.template(template -> { + template.aliases(getAlias(index), a -> a); + template.mappings(mapping -> { + mapping.dynamicTemplates(createDynamicTemplates()); + mapping.properties(createElasticProperties(metadata.getProperties())); + mapping.source(s -> s.enabled(true)); + return mapping; + }); + return template; + }); + return builder; } @Override public Mono loadIndexMetadata(String index) { - return client.getTemplate(new GetIndexTemplatesRequest(getTemplate(index))) - .filter(resp -> CollectionUtils.isNotEmpty(resp.getIndexTemplates())) - .flatMap(resp -> Mono.justOrEmpty(convertMetadata(index, resp - .getIndexTemplates() - .get(0) - .mappings()))); + String name = getTemplate(index); + return client.execute(t -> { + TemplateMapping mapping = ElasticSearchSupport + .current() + .getTemplateMapping( + t.indices() + .getTemplate(request -> request.name(getTemplate(index))), + name + ); + return mapping == null ? null : convertMetadata(index, mapping.mappings()); + }); } } diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/strategies/TimeByDayElasticSearchIndexStrategy.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/strategies/TimeByDayElasticSearchIndexStrategy.java old mode 100644 new mode 100755 similarity index 60% rename from jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/strategies/TimeByDayElasticSearchIndexStrategy.java rename to jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/strategies/TimeByDayElasticSearchIndexStrategy.java index 3ef3e5f1..5467587e --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/strategies/TimeByDayElasticSearchIndexStrategy.java +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/strategies/TimeByDayElasticSearchIndexStrategy.java @@ -1,32 +1,29 @@ package org.jetlinks.community.elastic.search.index.strategies; -import org.hswebframework.utils.time.DateFormatter; import org.jetlinks.community.elastic.search.index.ElasticSearchIndexProperties; import org.jetlinks.community.elastic.search.service.reactive.ReactiveElasticsearchClient; -import org.springframework.stereotype.Component; +import java.time.Clock; import java.time.LocalDate; -import java.util.Date; /** - * 按日期来划分索引策略 + * 按天来划分索引策略 * - * @author caizz - * @since 1.0 + * @author zhouhao + * @since 2.2 */ -@Component public class TimeByDayElasticSearchIndexStrategy extends TemplateElasticSearchIndexStrategy { + private static final Clock CLOCK = Clock.systemDefaultZone(); + public TimeByDayElasticSearchIndexStrategy(ReactiveElasticsearchClient client, ElasticSearchIndexProperties properties) { super("time-by-day", client, properties); } @Override public String getIndexForSave(String index) { - LocalDate now = LocalDate.now(); + LocalDate now = LocalDate.now(CLOCK); String idx = wrapIndex(index); - return idx + "_" + now.getYear() - + "-" + (now.getMonthValue() < 10 ? "0" : "") + now.getMonthValue() - + "-" + (now.getDayOfMonth() < 10 ? "0" : "") + now.getDayOfMonth(); + return idx + "_" + now.getYear() + "-" + now.getMonthValue() + "-" + now.getDayOfMonth(); } } diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/strategies/TimeByMonthElasticSearchIndexStrategy.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/strategies/TimeByMonthElasticSearchIndexStrategy.java similarity index 85% rename from jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/strategies/TimeByMonthElasticSearchIndexStrategy.java rename to jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/strategies/TimeByMonthElasticSearchIndexStrategy.java index 1971fd6b..70899740 100755 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/strategies/TimeByMonthElasticSearchIndexStrategy.java +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/strategies/TimeByMonthElasticSearchIndexStrategy.java @@ -3,6 +3,7 @@ package org.jetlinks.community.elastic.search.index.strategies; import org.jetlinks.community.elastic.search.index.ElasticSearchIndexProperties; import org.jetlinks.community.elastic.search.service.reactive.ReactiveElasticsearchClient; +import java.time.Clock; import java.time.LocalDate; /** @@ -13,13 +14,15 @@ import java.time.LocalDate; */ public class TimeByMonthElasticSearchIndexStrategy extends TemplateElasticSearchIndexStrategy { + private static final Clock CLOCK = Clock.systemDefaultZone(); + public TimeByMonthElasticSearchIndexStrategy(ReactiveElasticsearchClient client, ElasticSearchIndexProperties properties) { super("time-by-month", client, properties); } @Override public String getIndexForSave(String index) { - LocalDate now = LocalDate.now(); + LocalDate now = LocalDate.now(CLOCK); String idx = wrapIndex(index); return idx + "_" + now.getYear() + "-" + now.getMonthValue(); } diff --git a/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/strategies/TimeByWeekElasticSearchIndexStrategy.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/strategies/TimeByWeekElasticSearchIndexStrategy.java new file mode 100755 index 00000000..db4f4162 --- /dev/null +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/index/strategies/TimeByWeekElasticSearchIndexStrategy.java @@ -0,0 +1,35 @@ +package org.jetlinks.community.elastic.search.index.strategies; + +import org.jetlinks.community.elastic.search.index.ElasticSearchIndexProperties; +import org.jetlinks.community.elastic.search.service.reactive.ReactiveElasticsearchClient; + +import java.time.Clock; +import java.time.LocalDate; +import java.time.temporal.WeekFields; +import java.util.Locale; + +/** + * 按每年第n周来划分索引策略 + * + * @author zhouhao + * @since 2.3 + */ +public class TimeByWeekElasticSearchIndexStrategy extends TemplateElasticSearchIndexStrategy { + + private static final Clock CLOCK = Clock.systemDefaultZone(); + + private static final WeekFields FIELDS = WeekFields.of(Locale.getDefault()); + + public TimeByWeekElasticSearchIndexStrategy(ReactiveElasticsearchClient client, + ElasticSearchIndexProperties properties) { + super("time-by-week", client, properties); + } + + @Override + public String getIndexForSave(String index) { + LocalDate now = LocalDate.now(CLOCK); + String idx = wrapIndex(index); + + return idx + "_" + now.getYear() + "-woy-" + now.get(FIELDS.weekOfYear()); + } +} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/AggregationService.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/service/AggregationService.java old mode 100644 new mode 100755 similarity index 77% rename from jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/AggregationService.java rename to jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/service/AggregationService.java index 6a3d87bb..791c77ee --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/AggregationService.java +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/service/AggregationService.java @@ -11,6 +11,13 @@ import java.util.Map; **/ public interface AggregationService { + /** + * 聚合查询,相同分组的结果不会合并到一起,需要自行处理合并 + * + * @param index 索引 + * @param queryParam 聚合查询参数 + * @return 查询结果 + */ Flux> aggregation(String[] index, AggregationQueryParam queryParam); /** diff --git a/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/service/ElasticSearchService.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/service/ElasticSearchService.java new file mode 100755 index 00000000..8b38f3e1 --- /dev/null +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/service/ElasticSearchService.java @@ -0,0 +1,248 @@ +package org.jetlinks.community.elastic.search.service; + +import org.hswebframework.ezorm.core.param.QueryParam; +import org.hswebframework.web.api.crud.entity.PagerResult; +import org.hswebframework.web.bean.FastBeanCopier; +import org.jetlinks.community.elastic.search.index.ElasticSearchIndexManager; +import org.jetlinks.community.elastic.search.service.reactive.ReactiveElasticsearchClient; +import org.jetlinks.community.elastic.search.index.ElasticIndex; +import org.jetlinks.community.elastic.search.index.ElasticSearchIndexStrategy; +import org.reactivestreams.Publisher; +import reactor.core.publisher.Flux; +import reactor.core.publisher.GroupedFlux; +import reactor.core.publisher.Mono; + +import java.util.Collection; +import java.util.Map; +import java.util.function.Function; + +/** + * ElasticSearch 服务 + * + * @author zhouhao + * @see ElasticSearchIndexManager + * @see ReactiveElasticsearchClient + * @since 1.0 + */ +public interface ElasticSearchService { + + /** + * 根据索引动态分页查询数据 + * + * @param index 索引 + * @param queryParam 查询参数 + * @param mapper 结果转换器 + * @param 结果类型 + * @return 分页查询结果 + * @see ElasticSearchIndexStrategy + * @see ElasticSearchIndexStrategy#getIndexForSearch(String) + */ + default Mono> queryPager(String index, + QueryParam queryParam, + Function, T> mapper) { + return queryPager(new String[]{index}, queryParam, mapper); + } + + /** + * 分页查询多个索引数据 + * + * @param index 索引列表 + * @param queryParam 查询条件 + * @param mapper 结果转换器 + * @param 结果类型 + * @return 分页查询结果 + * @see ElasticSearchIndexStrategy + * @see ElasticSearchIndexStrategy#getIndexForSearch(String) + */ + Mono> queryPager(String[] index, QueryParam queryParam, Function, T> mapper); + + /** + * 查询数据 + * + * @param index 索引 + * @param queryParam 查询条件 + * @param mapper 结果转换器 + * @param 结果类型 + * @return 查询结果 + * @see ElasticSearchIndexStrategy + * @see ElasticSearchIndexStrategy#getIndexForSearch(String) + */ + Flux query(String index, QueryParam queryParam, Function, T> mapper); + + /** + * 查询多个索引数据 + * + * @param index 索引 + * @param queryParam 查询条件 + * @param mapper 结果转换器 + * @param 结果类型 + * @see ElasticSearchIndexStrategy + * @see ElasticSearchIndexStrategy#getIndexForSearch(String) + */ + Flux query(String[] index, QueryParam queryParam, Function, T> mapper); + + /** + * 根据多个条件,执行多次查询并一次性返回结果 + * + * @param index 索引 + * @param queryParam 查询条件 + * @param mapper 结果转换器 + * @param 结果类型 + * @return 查询结果 + * @see ElasticSearchIndexStrategy + * @see ElasticSearchIndexStrategy#getIndexForSearch(String) + */ + default Flux multiQuery(String index, Collection queryParam, Function, T> mapper) { + return multiQuery(new String[]{index}, queryParam, mapper); + } + + /** + * 根据多个条件,执行多次查询并一次性返回结果 + * + * @param index 索引 + * @param queryParam 查询条件 + * @param mapper 结果转换器 + * @param 结果类型 + * @return 查询结果 + * @see ElasticSearchIndexStrategy + * @see ElasticSearchIndexStrategy#getIndexForSearch(String) + */ + Flux multiQuery(String[] index, Collection queryParam, Function, T> mapper); + + /** + * 查询数据总数 + * + * @param index 索引 + * @param queryParam 查询条件 + * @return 查询结果 + * @see ElasticSearchIndexStrategy + * @see ElasticSearchIndexStrategy#getIndexForSearch(String) + */ + default Mono count(String index, QueryParam queryParam) { + return count(new String[]{index}, queryParam); + } + + /** + * 查询数据总数 + * + * @param index 索引 + * @param queryParam 查询条件 + * @return 查询结果 + * @see ElasticSearchIndexStrategy + * @see ElasticSearchIndexStrategy#getIndexForSearch(String) + */ + Mono count(String[] index, QueryParam queryParam); + + /** + * 按查询条件删除数据 + * + * @param index 索引 + * @param queryParam 查询条件 + * @return 查询结果 + */ + Mono delete(String index, QueryParam queryParam); + + /** + * 提交一个索引记录,此操作不会立即存储数据,将会进行本地缓冲,满足一定条件后批量写入es. + * + * @param index 索引 + * @param payload 记录值 + * @param 类型 + * @return void + */ + Mono commit(String index, T payload); + + /** + * 提交多个索引记录,此操作不会立即存储数据,将会进行本地缓冲,满足一定条件后批量写入es. + * + * @param index 索引 + * @param payload 记录值 + * @param 类型 + * @return void + */ + Mono commit(String index, Collection payload); + + /** + * 提交多个索引记录,此操作不会立即存储数据,将会进行本地缓冲,满足一定条件后批量写入es. + * + * @param index 索引 + * @param payload 记录值 + * @param 类型 + * @return void + */ + Mono commit(String index, Publisher payload); + + /** + * 立即保存索引记录 + * + * @param index 索引 + * @param payload 记录值 + * @param 类型 + * @return void + */ + Mono save(String index, T payload); + + /** + * 立即保存多个索引记录 + * + * @param index 索引 + * @param payload 记录值 + * @param 类型 + * @return void + */ + Mono save(String index, Collection payload); + + /** + * 立即保存多个记录 + * + * @param index 索引 + * @param payload 记录值 + * @param 类型 + * @return void + */ + Mono save(String index, Publisher payload); + + /*====== 查询并转换为指定的类型 ======*/ + + default Flux query(String index, QueryParam queryParam, Class type) { + return query(index, queryParam, map -> FastBeanCopier.copy(map, type)); + } + + default Mono> queryPager(String index, QueryParam queryParam, Class type) { + return queryPager(index, queryParam, map -> FastBeanCopier.copy(map, type)); + } + + default Mono> queryPager(ElasticIndex index, QueryParam queryParam, Class type) { + return queryPager(index.getIndex(), queryParam, type); + } + + default Mono> queryPager(ElasticIndex index, QueryParam queryParam, Function, T> mapper) { + return queryPager(index.getIndex(), queryParam, mapper); + } + + /*====== 使用索引定义执行响应到操作 ======*/ + default Flux query(ElasticIndex index, QueryParam queryParam, Class type) { + return query(index.getIndex(), queryParam, type); + } + + default Flux query(ElasticIndex index, QueryParam queryParam, Function, T> mapper) { + return this.query(index.getIndex(), queryParam, mapper); + } + + default Mono count(ElasticIndex index, QueryParam data) { + return this.count(index.getIndex(), data); + } + + default Mono commit(ElasticIndex index, T data) { + return this.commit(index.getIndex(), data); + } + + default Mono commit(ElasticIndex index, Collection data) { + return this.commit(index.getIndex(), data); + } + + default Mono commit(ElasticIndex index, Publisher data) { + return this.commit(index.getIndex(), data); + } + +} diff --git a/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/service/reactive/AggType.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/service/reactive/AggType.java new file mode 100755 index 00000000..e67463c4 --- /dev/null +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/service/reactive/AggType.java @@ -0,0 +1,171 @@ +package org.jetlinks.community.elastic.search.service.reactive; + +import co.elastic.clients.elasticsearch._types.FieldValue; +import co.elastic.clients.elasticsearch._types.aggregations.Aggregation; +import co.elastic.clients.json.JsonData; +import co.elastic.clients.util.ObjectBuilder; +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.hswebframework.ezorm.core.param.Term; +import org.hswebframework.ezorm.core.param.TermType; +import org.jetlinks.community.elastic.search.index.ElasticSearchIndexMetadata; +import org.jetlinks.community.elastic.search.utils.QueryParamTranslator; + +import java.util.Collections; + +@Getter +@AllArgsConstructor +public enum AggType { + + AVG("平均") { + @Override + public Aggregation.Builder.ContainerBuilder aggregationBuilder(String name, String field, + ElasticSearchIndexMetadata metadata, + Aggregation.Builder builder, + Object missing) { + return builder.avg(avg -> avg + .field(field) + .missing(missing == null ? null : FieldValue.of(JsonData.of(missing)))); + } + }, + MAX("最大") { + @Override + public Aggregation.Builder.ContainerBuilder aggregationBuilder(String name, String field, + ElasticSearchIndexMetadata metadata, + Aggregation.Builder builder, + Object missing) { + return builder.max(max -> max + .field(field) + .missing(missing == null ? null : FieldValue.of(JsonData.of(missing)))); + } + }, + MEDIAN("中间值") { + @Override + public Aggregation.Builder.ContainerBuilder aggregationBuilder(String name, String field, + ElasticSearchIndexMetadata metadata, + Aggregation.Builder builder, + Object missing) { + return builder.medianAbsoluteDeviation(m -> m + .field(field) + .missing(missing == null ? null : FieldValue.of(JsonData.of(missing)))); + } + }, + STDDEV("标准差") { + @Override + public Aggregation.Builder.ContainerBuilder aggregationBuilder(String name, String field, + ElasticSearchIndexMetadata metadata, + Aggregation.Builder builder, + Object missing) { + return builder.extendedStats(avg -> avg + .field(field) + .missing(missing == null ? null : FieldValue.of(JsonData.of(missing)))); + } + }, + COUNT("非空值计数") { + @Override + public Aggregation.Builder.ContainerBuilder aggregationBuilder(String name, String field, + ElasticSearchIndexMetadata metadata, + Aggregation.Builder builder, + Object missing) { + + return builder.filter(q -> QueryParamTranslator + .applyQueryBuilder(q, + Collections.singletonList(Term.of(field, TermType.notnull, field)), + metadata)); + } + }, + DISTINCT_COUNT("去重计数") { + @Override + public Aggregation.Builder.ContainerBuilder aggregationBuilder(String name, String field, + ElasticSearchIndexMetadata metadata, + Aggregation.Builder builder, + Object missing) { + + return builder.cardinality(cardinality -> cardinality + .field(field) + .missing(missing == null ? null : FieldValue.of(JsonData.of(missing)))); + } + }, + MIN("最小") { + @Override + public Aggregation.Builder.ContainerBuilder aggregationBuilder(String name, String field, + ElasticSearchIndexMetadata metadata, + Aggregation.Builder builder, + Object missing) { + + return builder.min(cardinality -> cardinality + .field(field) + .missing(missing == null ? null : FieldValue.of(JsonData.of(missing)))); + } + }, + FIRST("第一条数据") { + @Override + public Aggregation.Builder.ContainerBuilder aggregationBuilder(String name, String field, + ElasticSearchIndexMetadata metadata, + Aggregation.Builder builder, + Object missing) { + + return builder.topHits(top -> top + .size(1) + .missing(missing == null ? null : FieldValue.of(JsonData.of(missing)))); + } + }, + TOP("第N条数据") { + @Override + public Aggregation.Builder.ContainerBuilder aggregationBuilder(String name, + String field, + ElasticSearchIndexMetadata metadata, + Aggregation.Builder builder, + Object missing) { + + return builder.topHits(top -> top + .field(field) + .missing(missing == null ? null : FieldValue.of(JsonData.of(missing)))); + } + }, + SUM("总和") { + @Override + public Aggregation.Builder.ContainerBuilder aggregationBuilder(String name, String field, + ElasticSearchIndexMetadata metadata, + Aggregation.Builder builder, + Object missing) { + + return builder.sum(b -> b + .field(field) + .missing(missing == null ? null : FieldValue.of(JsonData.of(missing)))); + } + }, + STATS("统计汇总") { + @Override + public Aggregation.Builder.ContainerBuilder aggregationBuilder(String name, String field, + ElasticSearchIndexMetadata metadata, + Aggregation.Builder builder, + Object missing) { + + return builder.stats(b -> b + .field(field) + .missing(missing == null ? null : FieldValue.of(JsonData.of(missing)))); + } + + }; + + @Getter + private final String text; + + public abstract ObjectBuilder aggregationBuilder(String name, + String field, + ElasticSearchIndexMetadata metadata, + Aggregation.Builder builder, + Object missing); + + + public static AggType of(String name) { + for (AggType type : AggType.values()) { + if (type.name().equalsIgnoreCase(name)) { + return type; + } + } + throw new UnsupportedOperationException("不支持的聚合类型：" + name); + } + +} \ No newline at end of file diff --git a/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/service/reactive/DefaultReactiveElasticsearchClient.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/service/reactive/DefaultReactiveElasticsearchClient.java new file mode 100755 index 00000000..20d76b37 --- /dev/null +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/service/reactive/DefaultReactiveElasticsearchClient.java @@ -0,0 +1,67 @@ +package org.jetlinks.community.elastic.search.service.reactive; + +import co.elastic.clients.elasticsearch.ElasticsearchAsyncClient; +import co.elastic.clients.elasticsearch.ElasticsearchClient; +import co.elastic.clients.transport.Version; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.jetlinks.core.lang.SharedPathString; +import org.jetlinks.core.trace.MonoTracer; +import org.jetlinks.core.trace.TraceHolder; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; + +@Slf4j +public class DefaultReactiveElasticsearchClient implements ReactiveElasticsearchClient { + private final static SharedPathString TRACE_SPAN_NAME_EXECUTE = + SharedPathString.of("/DefaultReactiveElasticsearchClient/execute"); + private final static SharedPathString TRACE_SPAN_NAME_EXECUTE_ASYNC = + SharedPathString.of("/DefaultReactiveElasticsearchClient/executeAsync"); + + private final ElasticsearchClient client; + private final ElasticsearchAsyncClient asyncClient; + + private Version serverVersion; + + public DefaultReactiveElasticsearchClient(ElasticsearchClient client) { + this.client = client; + this.asyncClient = new ElasticsearchAsyncClient(client._transport(), client._transportOptions()); + } + + @Override + @SneakyThrows + public Version serverVersion() { + if (serverVersion != null) { + return serverVersion; + } + synchronized (this) { + return serverVersion = Version + .parse(client.info() + .version() + .number()); + } + + } + + @Override + public Mono executeAsync(ElasticsearchAsyncClientCallback callback) { + return Mono + .defer(() -> { + try { + return Mono + .fromCompletionStage(callback.execute(asyncClient)); + } catch (Exception e) { + return Mono.error(e); + } + }) + .as(MonoTracer.create(TRACE_SPAN_NAME_EXECUTE_ASYNC)); + } + + @Override + public Mono execute(ElasticsearchClientCallback callback) { + return Mono + .fromCallable(() -> callback.execute(client)) + .as(MonoTracer.create(TRACE_SPAN_NAME_EXECUTE)) + .subscribeOn(Schedulers.boundedElastic()); + } +} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/reactive/ElasticSearchBufferProperties.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/service/reactive/ElasticSearchBufferProperties.java similarity index 91% rename from jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/reactive/ElasticSearchBufferProperties.java rename to jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/service/reactive/ElasticSearchBufferProperties.java index ff6d334c..b134ee81 100644 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/reactive/ElasticSearchBufferProperties.java +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/service/reactive/ElasticSearchBufferProperties.java @@ -3,7 +3,6 @@ package org.jetlinks.community.elastic.search.service.reactive; import lombok.Getter; import lombok.Setter; import org.jetlinks.community.buffer.BufferProperties; -import org.jetlinks.community.buffer.BufferProperties; import org.springframework.boot.context.properties.ConfigurationProperties; @Getter @@ -17,4 +16,6 @@ public class ElasticSearchBufferProperties extends BufferProperties { } private boolean refreshWhenWrite = false; + + private boolean ignoreDocId = false; } \ No newline at end of file diff --git a/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/service/reactive/ReactiveAggregationService.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/service/reactive/ReactiveAggregationService.java new file mode 100755 index 00000000..8005101c --- /dev/null +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/service/reactive/ReactiveAggregationService.java @@ -0,0 +1,450 @@ +package org.jetlinks.community.elastic.search.service.reactive; + +import co.elastic.clients.elasticsearch._types.FieldValue; +import co.elastic.clients.elasticsearch._types.ScoreSort; +import co.elastic.clients.elasticsearch._types.SortOrder; +import co.elastic.clients.elasticsearch._types.aggregations.*; +import co.elastic.clients.elasticsearch._types.aggregations.Aggregation; +import co.elastic.clients.json.JsonData; +import co.elastic.clients.util.NamedValue; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.MapUtils; +import org.hswebframework.ezorm.core.param.QueryParam; +import org.hswebframework.ezorm.core.param.Sort; +import org.hswebframework.ezorm.core.param.Term; +import org.hswebframework.ezorm.core.param.TermType; +import org.hswebframework.web.api.crud.entity.QueryParamEntity; +import org.jetlinks.core.metadata.types.DateTimeType; +import org.jetlinks.community.Interval; +import org.jetlinks.community.elastic.search.ElasticSearchSupport; +import org.jetlinks.community.elastic.search.index.ElasticSearchIndexManager; +import org.jetlinks.community.elastic.search.index.ElasticSearchIndexMetadata; +import org.jetlinks.community.elastic.search.service.AggregationService; +import org.jetlinks.community.elastic.search.utils.QueryParamTranslator; +import org.jetlinks.community.timeseries.query.*; +import org.jetlinks.reactor.ql.utils.CastUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.util.StringUtils; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.util.function.Tuple2; +import reactor.util.function.Tuple3; +import reactor.util.function.Tuples; + +import java.time.Duration; +import java.time.ZoneId; +import java.util.*; + +/** + * @author zhouhao + * @since 1.5 + **/ +@Slf4j +public class ReactiveAggregationService implements AggregationService { + + private final ReactiveElasticsearchClient restClient; + + private final ElasticSearchIndexManager indexManager; + + //是否打开执行提示，打开后，聚合查询时会给每一个bucket构造global ordinals。单个索引数据量大于一百万时建议关闭。 + private static final boolean IS_OPEN_GLOBAL_ORDINALS = + Boolean.parseBoolean(System.getProperty("elasticsearch.agg.query.execution_hint", "false")); + + @Autowired + public ReactiveAggregationService(ElasticSearchIndexManager indexManager, + ReactiveElasticsearchClient restClient) { + this.restClient = restClient; + this.indexManager = indexManager; + } + + private Aggregation.Builder.ContainerBuilder createGroupAggregation(Group group, + AggregationQueryParam param) { + if (group instanceof TimeGroup timeGroup) { + Interval interval = timeGroup.getInterval(); + return new Aggregation.Builder() + .dateHistogram(his -> { + his.timeZone(ZoneId.systemDefault().toString()); + his.field(timeGroup.getProperty()); + his.order(Collections.singletonList( + NamedValue.of("_key", SortOrder.Desc) + )); + if (StringUtils.hasText(timeGroup.getFormat())) { + String format = timeGroup.getFormat(); + if (format.startsWith("yyyy")) { + format = "8" + format; + } + his.format(format); + } + if (interval.getNumber().intValue() == 1) { + his.calendarInterval(convertCalendarInterval(interval.getExpression())); + } else { + his.fixedInterval(t -> t.time(interval.toString())); + } + his.extendedBounds(bounds -> bounds + .min(FieldDateMath.of(b -> b.value((double) calculateStartWithTime(param)))) + .max(FieldDateMath.of(b -> b.value((double) param.getEndWithTime())))); + return his; + }); + } else { + return new Aggregation.Builder() + .terms(terms -> { + terms.field(group.getProperty()); + if (group instanceof LimitGroup) { + terms.size(((LimitGroup) group).getLimit()); + } else { + terms.size(100); + } + if (IS_OPEN_GLOBAL_ORDINALS) { + terms.executionHint(TermsAggregationExecutionHint.Map); + } + return terms; + }); + } + } + + private CalendarInterval convertCalendarInterval(String expression) { + return switch (expression) { + case Interval.year -> CalendarInterval.Year; + case Interval.quarter -> CalendarInterval.Quarter; + case Interval.month -> CalendarInterval.Month; + case Interval.weeks -> CalendarInterval.Week; + case Interval.days -> CalendarInterval.Day; + case Interval.hours -> CalendarInterval.Hour; + case Interval.minutes -> CalendarInterval.Minute; + case Interval.seconds -> CalendarInterval.Second; + default -> throw new UnsupportedOperationException("不支持的时间间隔:" + expression); + }; + } + + private Map createAggregations(ElasticSearchIndexMetadata metadata, + AggregationQueryParam param) { + + Map colAgg = Maps.newHashMapWithExpectedSize(param.getAggColumns().size()); + for (AggregationColumn aggColumn : param.getAggColumns()) { + if (aggColumn instanceof LimitAggregationColumn limitAggregationColumn) { + colAgg.put( + aggColumn.getAlias(), + Aggregation.of(builder -> builder + .topHits(top -> { + top + .size(limitAggregationColumn.getLimit()) + .missing(aggColumn.getDefaultValue() == null + ? null + : FieldValue.of(JsonData.of(aggColumn.getDefaultValue()))); + param.getQueryParam() + .getSorts() + .forEach(sort -> { + if (StringUtils.hasText(sort.getName())) { + top.sort(s -> s + .field(f -> f + .field(sort.getName()) + .order("desc".equalsIgnoreCase(sort.getOrder()) ? SortOrder.Desc : SortOrder.Asc))); + } + }); + return top; + })) + ); + continue; + } + colAgg.put( + aggColumn.getAlias(), + AggType + .of(aggColumn.getAggregation().name()) + .aggregationBuilder(aggColumn.getAlias(), + aggColumn.getProperty(), + metadata, + new Aggregation.Builder(), + aggColumn.getDefaultValue()) + .build() + ); + } + + //不分组 + List groups = param.getGroups(); + if (CollectionUtils.isEmpty(groups)) { + return colAgg; + } + Tuple2 frist = null; + for (int i = groups.size() - 1; i >= 0; i--) { + Group group = groups.get(i); + Aggregation agg; + if (frist == null) { + //最后一个分组添加列聚合 + agg = createGroupAggregation(group, param).aggregations(colAgg).build(); + } else { + agg = createGroupAggregation(group, param) + .aggregations(Collections.singletonMap(frist.getT1(), frist.getT2())) + .build(); + //添加上一个分组 + } + frist = Tuples.of(group.getAlias(), agg); + } + + if (frist == null) { + return colAgg; + } + + return Collections.singletonMap(frist.getT1(), frist.getT2()); + + } + + @Override + public Flux> aggregation(String index, AggregationQueryParam aggregationQueryParam) { + boolean isGroup = CollectionUtils.isNotEmpty(aggregationQueryParam.getGroups()); + + QueryParamEntity param = aggregationQueryParam.getQueryParam().clone(); + param + .toQuery() + .between(aggregationQueryParam.getTimeProperty(), + aggregationQueryParam.getStartWithTime(), + aggregationQueryParam.getEndWithTime()); + + + return Mono.zip( + Mono.just(index), + indexManager + .getIndexStrategy(index) + .map(s -> s.getIndexForSearch(index)), + indexManager.getIndexMetadata(index)) + .flatMapMany(tps -> { + ElasticSearchIndexMetadata metadata = tps.getT3(); + return restClient + .execute(client -> client + .search(search -> search + .index(tps.getT2()) + .query(q -> QueryParamTranslator.applyQueryBuilder(q, param, metadata)) + .size(0) + .aggregations(createAggregations(metadata, aggregationQueryParam)), Map.class)) + .flatMapMany(resp -> Flux + .fromIterable(resp.aggregations().entrySet()) + .concatMap(e -> parseAggregation(e.getKey(), e.getValue()))) + .as(flux -> { + if (!isGroup) { + return flux + .map(Map::entrySet) + .flatMap(Flux::fromIterable) + .collectMap(Map.Entry::getKey, Map.Entry::getValue) + .flux(); + } + return flux; + }); + }); + } + + @Override + public Flux> aggregation(String[] index, AggregationQueryParam aggregationQueryParam) { + if (index.length == 1) { + return aggregation(index[0], aggregationQueryParam); + } + boolean isGroup = CollectionUtils.isNotEmpty(aggregationQueryParam.getGroups()); + return Flux.fromArray(index) + .flatMap(idx -> + Mono.zip(Mono.just(idx), + indexManager + .getIndexStrategy(idx) + .map(s -> s.getIndexForSearch(idx)), + indexManager.getIndexMetadata(idx))) + .collectList() + .flatMapMany(tps -> { + ElasticSearchIndexMetadata metadata = tps.get(0).getT3(); + return restClient + .execute(client -> client + .search(search -> search + .index(Lists.transform(tps, Tuple3::getT2)) + .query(q -> QueryParamTranslator.applyQueryBuilder(q, aggregationQueryParam.getQueryParam(), metadata)) + .size(0) + .aggregations(createAggregations(metadata, aggregationQueryParam)), Map.class)) + .flatMapMany(resp -> Flux + .fromIterable(resp.aggregations().entrySet()) + .concatMap(e -> parseAggregation(e.getKey(), e.getValue()))) + .as(flux -> { + if (!isGroup) { + return flux + .map(Map::entrySet) + .flatMap(Flux::fromIterable) + .collectMap(Map.Entry::getKey, Map.Entry::getValue) + .flux(); + } + return flux; + }); + }); + + } + + private Flux> parseAggregation(String name, + Aggregate aggregate) { + if (aggregate.isSum()) { + return Flux.just(Collections.singletonMap(name, getSafeNumber(aggregate.sum().value()))); + } + if (aggregate.isAvg()) { + return Flux.just(Collections.singletonMap(name, getSafeNumber(aggregate.avg().value()))); + } + if (aggregate.isMax()) { + return Flux.just(Collections.singletonMap(name, getSafeNumber(aggregate.max().value()))); + } + if (aggregate.isMin()) { + return Flux.just(Collections.singletonMap(name, getSafeNumber(aggregate.min().value()))); + } + if (aggregate.isFilter()) { + return Flux.just(Collections.singletonMap(name, aggregate.filter().docCount())); + } + if (aggregate.isValueCount()) { + return Flux.just(Collections.singletonMap(name, getSafeNumber(aggregate.valueCount().value()))); + } + if (aggregate.isSterms()) { + return parseAggregation(name, aggregate.sterms()); + } + if (aggregate.isLterms()) { + return parseAggregation(name, aggregate.lterms()); + } + if (aggregate.isDterms()) { + return parseAggregation(name, aggregate.dterms()); + } + if (aggregate.isSimpleValue()) { + return Flux.just(Collections.singletonMap(name, getSafeNumber(aggregate.simpleValue().value()))); + } + + if (aggregate.isExtendedStats()) { + //只处理了标准差 + return Flux.just(Collections.singletonMap(name, getSafeNumber(aggregate.extendedStats().stdDeviation()))); + } + + if (aggregate.isHistogram()) { + return parseAggregation(name, aggregate.histogram()); + } + + if (aggregate.isTopHits()) { + return Flux + .fromIterable(aggregate.topHits().hits().hits()) + .map(hit -> { + Map val = hit.source().to(Map.class); + if (!val.containsKey("id")) { + val.put("id", hit.id()); + } + return val; + }); + } + if (aggregate.isDateHistogram()) { + return parseAggregation(name, aggregate.dateHistogram()); + } + if (aggregate.isCardinality()) { + return Flux.just(Collections.singletonMap(name, aggregate.cardinality().value())); + } + log.warn("unsupported aggregation {} : {}", aggregate._kind(), aggregate); + return Flux.empty(); + } + + private Object getSafeNumber(double number) { + return (Double.isNaN(number) || Double.isInfinite(number)) ? null : number; + } + + private Flux> parseAggregation(String name, MultiBucketAggregateBase aggregation) { + Buckets buckets = aggregation.buckets(); + Flux bucketFlux; + if (buckets.isArray()) { + bucketFlux = Flux.fromIterable(buckets.array()); + } else { + bucketFlux = Flux.fromIterable(buckets.keyed().values()); + } + return bucketFlux + .concatMap(bucket -> Flux + .fromIterable(bucket.aggregations().entrySet()) + .concatMap(e -> parseAggregation(e.getKey(), e.getValue()), 0) + .map(map -> transformBucket(name, map, bucket)), 0); + + } + + private Map transformBucket(String name, + Map map, + MultiBucketBase bucket) { + return ElasticSearchSupport + .current() + .transformBucket(name, map, bucket); + } + + + private Object parseBucket(Object bucket) { + if (bucket instanceof LongTermsBucket) { + return ((LongTermsBucket) bucket).key(); + } + if (bucket instanceof DoubleTermsBucket) { + return ((DoubleTermsBucket) bucket).key(); + } + if (bucket instanceof StringTermsBucket) { + return ((StringTermsBucket) bucket).key()._get(); + } + return null; + } + + private Flux> parseAggregation(String name, TermsAggregateBase aggregation) { + Buckets buckets = aggregation.buckets(); + Flux flux; + if (buckets.isKeyed()) { + flux = Flux.fromIterable(buckets.keyed().values()); + } else { + flux = Flux.fromIterable(buckets.array()); + } + + return flux.concatMap(base -> Flux + .fromIterable(base.aggregations().entrySet()) + .concatMap(e -> parseAggregation(e.getKey(), e.getValue()), 0) + .map(map -> { + Map val = new HashMap<>(map); + val.put(name, parseBucket(base)); + return val; + }), 0); + } + + protected static QueryParam prepareQueryParam(AggregationQueryParam param) { + QueryParam queryParam = param.getQueryParam().clone(); + queryParam.setPaging(false); + boolean hasTimestamp = false; + for (Term term : queryParam.getTerms()) { + if (param.getTimeProperty().equals(term.getColumn())) { + hasTimestamp = true; + } + } + if (!hasTimestamp) { + queryParam.and(param.getTimeProperty(), TermType.btw, Arrays.asList(calculateStartWithTime(param), param.getEndWithTime())); + } + if (queryParam.getSorts().isEmpty()) { + queryParam.orderBy(param.getTimeProperty()).desc(); + } + return queryParam; + } + + //聚合查询默认的时间间隔 + static long thirtyDayMillis = Duration + .ofDays(Integer.getInteger("elasticsearch.agg.default-range-day", 90)) + .toMillis(); + + static long calculateStartWithTime(AggregationQueryParam param) { + long startWithParam = param.getStartWithTime(); + if (startWithParam == 0) { + //从查询条件中提取时间参数来获取时间区间 + List terms = param.getQueryParam().getTerms(); + for (Term term : terms) { + if ("timestamp".equals(term.getColumn())) { + Object value = term.getValue(); + String termType = term.getTermType(); + if (TermType.btw.equals(termType)) { + if (String.valueOf(value).contains(",")) { + value = Arrays.asList(String.valueOf(value).split(",")); + } + return DateTimeType.GLOBAL.convert(CastUtils.castArray(value).get(0)).getTime(); + } + if (TermType.gt.equals(termType) || TermType.gte.equals(termType)) { + + return DateTimeType.GLOBAL.convert(value).getTime(); + } + } + } + return param.getEndWithTime() - thirtyDayMillis; + } + return startWithParam; + } + +} \ No newline at end of file diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/reactive/ReactiveElasticSearchService.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/service/reactive/ReactiveElasticSearchService.java similarity index 53% rename from jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/reactive/ReactiveElasticSearchService.java rename to jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/service/reactive/ReactiveElasticSearchService.java index 0c8d5643..673451e6 100755 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/reactive/ReactiveElasticSearchService.java +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/service/reactive/ReactiveElasticSearchService.java @@ -1,72 +1,64 @@ package org.jetlinks.community.elastic.search.service.reactive; +import co.elastic.clients.elasticsearch._types.ElasticsearchException; +import co.elastic.clients.elasticsearch._types.ErrorResponse; +import co.elastic.clients.elasticsearch._types.Refresh; +import co.elastic.clients.elasticsearch._types.Time; +import co.elastic.clients.elasticsearch.core.BulkResponse; +import co.elastic.clients.elasticsearch.core.SearchRequest; +import co.elastic.clients.elasticsearch.core.SearchResponse; +import co.elastic.clients.elasticsearch.core.bulk.BulkOperation; +import co.elastic.clients.elasticsearch.core.bulk.BulkResponseItem; +import co.elastic.clients.elasticsearch.core.bulk.IndexOperation; +import co.elastic.clients.elasticsearch.core.msearch.MultiSearchItem; +import co.elastic.clients.elasticsearch.core.msearch.MultiSearchResponseItem; +import co.elastic.clients.elasticsearch.core.search.Hit; +import co.elastic.clients.elasticsearch.core.search.HitsMetadata; +import co.elastic.clients.elasticsearch.core.search.TrackHits; +import com.alibaba.fastjson.JSONObject; import com.google.common.collect.Collections2; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import io.opentelemetry.api.common.AttributeKey; import lombok.Getter; import lombok.Setter; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.Version; -import org.elasticsearch.action.bulk.BulkItemResponse; -import org.elasticsearch.action.bulk.BulkRequest; -import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.search.MultiSearchRequest; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.action.support.WriteRequest; -import org.elasticsearch.common.Strings; -import org.elasticsearch.core.TimeValue; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.reindex.BulkByScrollResponse; -import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.xcontent.XContentType; import org.hswebframework.ezorm.core.param.QueryParam; -import org.hswebframework.utils.time.DateFormatter; -import org.hswebframework.utils.time.DefaultDateFormatter; +import org.hswebframework.utils.StringUtils; import org.hswebframework.web.api.crud.entity.PagerResult; +import org.hswebframework.web.api.crud.entity.QueryParamEntity; import org.hswebframework.web.bean.FastBeanCopier; import org.hswebframework.web.exception.BusinessException; -import org.jetlinks.community.buffer.*; +import org.jetlinks.community.configure.cluster.Cluster; +import org.jetlinks.core.metadata.Jsonable; import org.jetlinks.core.trace.MonoTracer; import org.jetlinks.core.utils.SerializeUtils; +import org.jetlinks.community.buffer.*; import org.jetlinks.community.elastic.search.index.ElasticSearchIndexManager; import org.jetlinks.community.elastic.search.index.ElasticSearchIndexMetadata; import org.jetlinks.community.elastic.search.service.ElasticSearchService; -import org.jetlinks.community.elastic.search.utils.ElasticSearchConverter; import org.jetlinks.community.elastic.search.utils.QueryParamTranslator; import org.jetlinks.community.utils.ErrorUtils; -import org.jetlinks.community.utils.ObjectMappers; import org.jetlinks.community.utils.SystemUtils; import org.reactivestreams.Publisher; import org.springframework.boot.CommandLineRunner; import org.springframework.boot.SpringApplication; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.context.annotation.DependsOn; import org.springframework.http.HttpStatus; -import org.springframework.stereotype.Service; -import org.springframework.util.StringUtils; +import org.springframework.util.ObjectUtils; import org.springframework.web.reactive.function.client.WebClientException; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -import reactor.core.scheduler.Schedulers; import reactor.util.function.Tuple2; import reactor.util.function.Tuples; -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; import java.io.Externalizable; import java.io.IOException; import java.io.ObjectInput; import java.io.ObjectOutput; import java.util.*; -import java.util.concurrent.TimeoutException; import java.util.function.Function; -import java.util.regex.Pattern; import java.util.stream.Collectors; /** @@ -77,23 +69,15 @@ import java.util.stream.Collectors; * @since 1.0 **/ @Slf4j -@DependsOn("reactiveElasticsearchClient") -@ConfigurationProperties(prefix = "elasticsearch") public class ReactiveElasticSearchService implements ElasticSearchService, CommandLineRunner { + static AttributeKey BUFFER_SIZE = AttributeKey.longKey("bufferSize"); + @Getter private final ReactiveElasticsearchClient restClient; @Getter private final ElasticSearchIndexManager indexManager; - public static final IndicesOptions indexOptions = IndicesOptions.fromOptions( - true, true, false, false - ); - - static { - DateFormatter.supportFormatter.add(new DefaultDateFormatter(Pattern.compile("[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.+"), "yyyy-MM-dd'T'HH:mm:ss.SSSZ")); - DateFormatter.supportFormatter.add(new DefaultDateFormatter(Pattern.compile("[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}.+"), "yyyy-MM-dd HH:mm:ss.SSS")); - } private PersistenceBuffer writer; @@ -117,6 +101,9 @@ public class ReactiveElasticSearchService implements ElasticSearchService, Comma @Override public Flux multiQuery(String[] index, Collection queryParam, Function, T> mapper) { + if (index.length == 1 && queryParam.size() == 1) { + return query(index[0], queryParam.iterator().next(), mapper); + } return indexManager .getIndexesMetadata(index) .flatMap(idx -> Mono.zip( @@ -124,27 +111,26 @@ public class ReactiveElasticSearchService implements ElasticSearchService, Comma )) .take(1) .singleOrEmpty() - .flatMapMany(indexMetadata -> { - MultiSearchRequest request = new MultiSearchRequest(); - return Flux - .fromIterable(queryParam) - .flatMap(entry -> createSearchRequest(entry, index)) - .doOnNext(request::add) - .then(Mono.just(request)) - .flatMapMany(searchRequest -> restClient - .multiSearch(searchRequest) - .flatMapMany(response -> Flux.fromArray(response.getResponses())) - .flatMap(item -> { - if (item.isFailure()) { - log.warn(item.getFailureMessage(), item.getFailure()); - return Mono.empty(); - } - return Flux - .fromIterable(translate((map) -> mapper - .apply(indexMetadata.getT1().convertFromElastic(map)), item.getResponse())); - })) - ; - }); + .flatMapMany(indexMetadata -> restClient + .execute(client -> client + .msearch(b -> { + for (QueryParam param : queryParam) { + b.searches(s -> s + .header(header -> header + .index(Arrays.asList(index)) + .ignoreUnavailable(true) + .allowNoIndices(true)) + .body(q -> { + QueryParamTranslator.convertSearchRequestBuilder(q, param, indexMetadata.getT1()); + return q; + })); + } + + return b; + }, Map.class) + ) + .flatMapMany(response -> translate((map) -> mapper + .apply(indexMetadata.getT1().convertFromElastic(map)), response.responses()))); } public Flux query(String index, QueryParam queryParam, Function, T> mapper) { @@ -157,9 +143,7 @@ public class ReactiveElasticSearchService implements ElasticSearchService, Comma .doQuery(index, queryParam) .flatMapMany(tp2 -> convertQueryResult(tp2.getT1(), tp2.getT2(), mapper)); } - return this - .doScrollQuery(index, queryParam) - .flatMap(tp2 -> convertQueryHit(tp2.getT1(), tp2.getT2(), mapper)); + return this.doScrollQuery(index, queryParam, mapper); } @Override @@ -181,77 +165,71 @@ public class ReactiveElasticSearchService implements ElasticSearchService, Comma .filter(CollectionUtils::isNotEmpty) .map(list -> PagerResult.of((int) tp2 .getT2() - .getHits() - .getTotalHits().value, list, queryParam)) + .hits().total().value(), list, queryParam)) ) .switchIfEmpty(Mono.fromSupplier(PagerResult::empty)); } private Flux convertQueryResult(List indexList, - SearchResponse response, + SearchResponse response, Function, T> mapper) { Map metadata = indexList .stream() .collect(Collectors.toMap(ElasticSearchIndexMetadata::getIndex, Function.identity())); return Flux - .fromIterable(response.getHits()) + .fromIterable(response.hits().hits()) .mapNotNull(hit -> { - Map hitMap = hit.getSourceAsMap(); - hitMap.putIfAbsent("id", hit.getId()); + Map hitMap = hit.source(); + if (hitMap == null) { + return null; + } + hitMap.putIfAbsent("id", hit.id()); return mapper .apply(Optional - .ofNullable(metadata.get(hit.getIndex())).orElse(indexList.get(0)) + .ofNullable(metadata.get(hit.index())).orElse(indexList.get(0)) .convertFromElastic(hitMap)); }); } - private Flux convertQueryHit(List indexList, - SearchHit searchHit, - Function, T> mapper) { - Map metadata = indexList - .stream() - .collect(Collectors.toMap(ElasticSearchIndexMetadata::getIndex, Function.identity())); - - return Flux - .just(searchHit) - .mapNotNull(hit -> { - Map hitMap = hit.getSourceAsMap(); - hitMap.putIfAbsent("id", hit.getId()); - return mapper - .apply(Optional - .ofNullable(metadata.get(hit.getIndex())).orElse(indexList.get(0)) - .convertFromElastic(hitMap)); - }); - - } - - private Mono, SearchResponse>> doQuery(String[] index, - QueryParam queryParam) { + private Mono, SearchResponse>> + doQuery(String[] index, + QueryParam queryParam) { return indexManager .getIndexesMetadata(index) .collectList() .filter(CollectionUtils::isNotEmpty) .flatMap(metadataList -> this .createSearchRequest(queryParam, metadataList) - .flatMap(restClient::searchForPage) + .flatMap(request -> restClient + .execute(c -> c.search(request, Map.class))) .map(response -> Tuples.of(metadataList, response)) ) ; } - private Flux, SearchHit>> doScrollQuery(String[] index, - QueryParam queryParam) { + private Flux doScrollQuery(String[] index, + QueryParam queryParam, + Function, T> mapper) { + Time time = Time.of(t -> t.time("10m")); + return indexManager .getIndexesMetadata(index) .collectList() .filter(CollectionUtils::isNotEmpty) .flatMapMany(metadataList -> this .createSearchRequest(queryParam.clone().noPaging(), metadataList) - .doOnNext(search -> search.source().size(getNoPagingPageSize(queryParam))) - .flatMapMany(restClient::scroll) - .map(searchHit -> Tuples.of(metadataList, searchHit)) + .flatMapMany(search -> new ScrollingFlux( + mapper, + restClient, + time, + builder -> { + builder.index(search.index()); + builder.query(search.query()); + builder.sort(search.sort()); + builder.size(getNoPagingPageSize(queryParam)); + })) ); } @@ -274,43 +252,38 @@ public class ReactiveElasticSearchService implements ElasticSearchService, Comma return this .getIndexForSearch(index) .flatMap(inx -> this - .createQueryBuilder(queryParam, index) - .flatMap(request -> restClient.deleteBy(delete -> delete.setQuery(request).indices(inx))) - .map(BulkByScrollResponse::getDeleted)) - .defaultIfEmpty(0L); + .createSearchRequest(queryParam, inx) + .flatMap(request -> restClient.execute( + client -> client + .deleteByQuery(q -> q.query(request.query()) + .index(request.index())) + .deleted())) + .defaultIfEmpty(0L)); } - private boolean checkWritable(String index) { -// if (SystemUtils.memoryIsOutOfWatermark()) { -// SystemUtils.printError("JVM内存不足,elasticsearch无法处理更多索引[%s]请求!", index); -// return false; -// } - return true; + private Map convertToMap(Object payload) { + @SuppressWarnings("all") + Map map = payload instanceof Map + ? ((Map) payload) : + FastBeanCopier.copy(payload, HashMap::new); + + return map; } @Override public Mono commit(String index, T payload) { - if (checkWritable(index)) { - writer.write(Buffer.of(index, payload)); - } - return Mono.empty(); + return writer + .writeAsync(Buffer.of(index, convertToMap(payload))); } @Override public Mono commit(String index, Collection payload) { - if (checkWritable(index)) { - for (T t : payload) { - writer.write(Buffer.of(index, t)); - } - } - return Mono.empty(); + return writer + .writeAsync(Collections2.transform(payload, t -> Buffer.of(index, convertToMap(t)))); } @Override public Mono commit(String index, Publisher data) { - if (!checkWritable(index)) { - return Mono.empty(); - } return Flux.from(data) .flatMap(d -> commit(index, d)) .then(); @@ -324,7 +297,7 @@ public class ReactiveElasticSearchService implements ElasticSearchService, Comma @Override public Mono save(String index, Publisher data) { return Flux.from(data) - .map(v -> Buffer.of(index, v)) + .map(v -> Buffer.of(index, convertToMap(v))) .buffer(buffer.getSize()) .flatMap(this::doSave) .then(); @@ -335,13 +308,20 @@ public class ReactiveElasticSearchService implements ElasticSearchService, Comma return save(index, Flux.fromIterable(payload)); } - @PreDestroy + @Override + public void run(String... args) { + startup(); + } + + public void shutdown0() { + writer.dispose(); + } + public void shutdown() { writer.stop(); } - @Override - public void run(String... args) throws Exception { + public void startup() { //spring 启动后更新配置信息 writer .settings(bufferSettings -> bufferSettings.properties(buffer)) @@ -350,55 +330,30 @@ public class ReactiveElasticSearchService implements ElasticSearchService, Comma //最后 shutdown SpringApplication .getShutdownHandlers() - .add(writer::dispose); - } - - - @Getter - @Setter - public static class BufferConfig extends BufferProperties { - public BufferConfig() { - //固定缓冲文件目录 - setFilePath("./data/elasticsearch-buffer"); - setSize(3000); - } - - private boolean refreshWhenWrite = false; - } - - @PostConstruct - public void reset() { - //spring 启动后更新配置信息 - writer.settings(bufferSettings -> bufferSettings.properties(buffer)); + .add(this::shutdown0); } private void init() { writer = new PersistenceBuffer<>( - BufferSettings.create("writer.queue", buffer), + BufferSettings.create(Cluster.safeId() + ".queue", buffer), Buffer::new, this::doSaveBuffer) .name("elasticsearch") .retryWhenError(e -> { - if (e instanceof ElasticsearchException) { - ElasticsearchException elasticsearchException = (ElasticsearchException) e; - if (elasticsearchException.status() == RestStatus.BAD_GATEWAY) { + if (e instanceof ElasticsearchException elasticsearchException) { + if (elasticsearchException.status() == 502) { return true; } } - return ErrorUtils.hasException( - e, - WebClientException.class, - IOException.class, - TimeoutException.class, - io.netty.handler.timeout.TimeoutException.class); + return ErrorUtils.hasException(e, WebClientException.class) + || ErrorUtils.hasException(e, IOException.class); }); writer.init(); } - public Mono doSaveBuffer(Collection> bufferFlux, PersistenceBuffer.FlushContext context) { List> list = bufferFlux instanceof List @@ -409,33 +364,36 @@ public class ReactiveElasticSearchService implements ElasticSearchService, Comma .doSave0(Collections2.transform(list, Buffered::getData)) .map(response -> { boolean hasError = false; - BulkItemResponse[] arr = response.getItems(); + List arr = response.items(); Set errors = null; + int responseSize = arr.size(); //响应数量不一致? - if (arr.length != size) { + if (responseSize != size) { log.warn("ElasticSearch response item size not equals to buffer size," + " response size:{}, buffer size:{}", - arr.length, + responseSize, size); } - for (int i = 0; i < arr.length; i++) { - BulkItemResponse item = arr[i]; + for (int i = 0; i < responseSize; i++) { + BulkResponseItem item = arr.get(i); Buffered buffered = size > i ? list.get(i) : null; - HttpStatus status = HttpStatus.resolve(item.status().getStatus()); + HttpStatus status = HttpStatus.resolve(item.status()); if ((status == null || !status.is2xxSuccessful())) { hasError = true; - if (null != item.getFailure()) { - context.error(new BusinessException.NoStackTrace(item.getFailure().getMessage())); + if (null != item.error()) { + context.error(new BusinessException.NoStackTrace(item.error().reason())); + } else { + context.error(new BusinessException.NoStackTrace(item.toString())); } if (log.isInfoEnabled()) { - String msg = item.getFailureMessage(); + String msg = item.error().reason(); if (errors == null) { errors = new HashSet<>(); } if (msg == null || errors.add(msg)) { log.info("write elasticsearch data [{}] failed: {}", buffered, - Strings.toString(item)); + item); } } //失败 @@ -461,80 +419,38 @@ public class ReactiveElasticSearchService implements ElasticSearchService, Comma } - private static final EnumSet deadStatus = EnumSet.of( - RestStatus.FORBIDDEN, - RestStatus.BAD_REQUEST, - RestStatus.UNAUTHORIZED, - RestStatus.NOT_FOUND, - RestStatus.METHOD_NOT_ALLOWED); + private static final Set deadStatus = Sets.newHashSet(403, 400, 401, 404, 405); - private boolean isDead(Buffered buffered, BulkItemResponse response) { + private boolean isDead(Buffered buffered, BulkResponseItem response) { return buffer.isExceededRetryCount(buffered.getRetryTimes()) || //快速失败,不再重试 deadStatus.contains(response.status()); } - protected Mono doSave0(Collection buffers) { - return Flux - .fromIterable(buffers) - .groupBy(Buffer::getIndex, Integer.MAX_VALUE) - .flatMap(group -> { - String index = group.key(); - return this - .getIndexForSave(index) - .flatMapMany(realIndex -> group - .map(buffer -> { - try { - IndexRequest request; - if (buffer.id != null) { - request = new IndexRequest(realIndex).id(buffer.id); - } else { - request = new IndexRequest(realIndex); - } - if (getRestClient().serverVersion().before(Version.V_7_0_0)) { - @SuppressWarnings("all") - IndexRequest ignore = request.type("_doc"); - } - request.source(buffer.payload, XContentType.JSON); - return request; - } finally { - buffer.release(); - } - })); - }) - .collectList() - .filter(CollectionUtils::isNotEmpty) - .flatMap(lst -> { - BulkRequest request = new BulkRequest(); - request.timeout(TimeValue.timeValueSeconds(9)); - if (buffer.isRefreshWhenWrite()) { - request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - } - lst.forEach(request::add); - return restClient.bulk(request); - }); - } - - @Getter - public static class Buffer implements Externalizable, MemoryUsage { + public static class Buffer implements Externalizable, MemoryUsage, Jsonable { private static final long serialVersionUID = 1; String index; String id; - byte[] payload; + Object payload; + + @Override + public JSONObject toJson() { + JSONObject object = new JSONObject(); + object.put("index", index); + object.put("id", id); + object.put("payload", payload); + return object; + } @SneakyThrows - public static Buffer of(String index, Object payload) { + public static Buffer of(String index, Map data) { Buffer buffer = new Buffer(); buffer.index = index; - @SuppressWarnings("unchecked") - Map data = payload instanceof Map - ? ((Map) payload) : - FastBeanCopier.copy(payload, HashMap::new); Object id = data.get("id"); buffer.id = id == null ? null : String.valueOf(id); - buffer.payload = ObjectMappers.JSON_MAPPER.writeValueAsBytes(data); + buffer.payload = data; return buffer; } @@ -546,22 +462,24 @@ public class ReactiveElasticSearchService implements ElasticSearchService, Comma public void writeExternal(ObjectOutput out) throws IOException { out.writeUTF(index); SerializeUtils.writeNullableUTF(id, out); - out.writeInt(payload.length); - out.write(payload); + SerializeUtils.writeObject(payload, out); } @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { index = in.readUTF(); id = SerializeUtils.readNullableUTF(in); - int length = in.readInt(); - payload = new byte[length]; - in.readFully(payload); + payload = SerializeUtils.readObject(in); } @Override public int usage() { - return payload == null ? 64 : 64 + payload.length; + return 1024; + } + + @Override + public String toString() { + return "index:" + index + ",id:" + id; } } @@ -579,42 +497,128 @@ public class ReactiveElasticSearchService implements ElasticSearchService, Comma } + protected Mono doSave0(Collection buffers) { + long size = buffers.size(); + return Flux + .fromIterable(buffers) + .groupBy(Buffer::getIndex, Integer.MAX_VALUE) + .flatMap(group -> { + String index = group.key(); + return this + .getIndexForSave(index) + .flatMapMany(realIndex -> group + .map(buffer -> IndexOperation + .of(b -> { + if (buffer.id != null && !this.buffer.isIgnoreDocId()) { + b.id(buffer.id); + } + b.index(realIndex); + b.document(buffer.payload); + return b; + }))); + }) + .collectList() + .filter(CollectionUtils::isNotEmpty) + .flatMap(lst -> restClient + .execute(c -> c.bulk(builder -> { + builder + .refresh(Refresh.True) + .operations(Lists.transform( + lst, v -> BulkOperation.of(b -> b.index(v)) + )); + return builder; + }))) + .as(MonoTracer.create( + "/_elasticsearch/save-buffer", + builder -> builder.setAttribute(BUFFER_SIZE, size))); + } + protected Mono doSave(Collection buffers) { return doSave0(buffers) .doOnError((err) -> { //这里的错误都输出到控制台,输入到slf4j可能会造成日志递归. SystemUtils.printError("保存ElasticSearch数据失败:\n%s", () -> new Object[]{ - org.hswebframework.utils.StringUtils.throwable2String(err) + StringUtils.throwable2String(err) }); }) .map(response -> { int success = 0; - for (BulkItemResponse item : response.getItems()) { - if (!item.isFailed()) { + for (BulkResponseItem item : response.items()) { + if (item.error() != null) { success++; } } return success; - }); + }) + ; } - private List translate(Function, T> mapper, SearchResponse response) { - return Arrays.stream(response.getHits().getHits()) - .map(hit -> { - Map hitMap = hit.getSourceAsMap(); - if (StringUtils.isEmpty(hitMap.get("id"))) { - hitMap.put("id", hit.getId()); - } - return mapper.apply(hitMap); - }) - .collect(Collectors.toList()); + private Exception createError(ErrorResponse response) { + + throw new ElasticsearchException(null, response); + } + + + @SuppressWarnings("all") + private Flux translate(Function, T> mapper, + HitsMetadata hits) { + return Flux.create(sink -> { + for (Hit hit : hits.hits()) { + if (sink.isCancelled()) { + break; + } + Map hitMap = hit.source(); + if (ObjectUtils.isEmpty(hitMap.get("id"))) { + hitMap.put("id", hit.id()); + } + sink.next(mapper.apply(hitMap)); + } + sink.complete(); + }); + } + + @SuppressWarnings("all") + private Flux translate(Function, T> mapper, + List> response) { + + return Flux.create(sink -> { + for (MultiSearchResponseItem mapMultiSearchResponseItem : response) { + if (mapMultiSearchResponseItem.isFailure()) { + sink.error(createError(mapMultiSearchResponseItem.failure())); + return; + } + MultiSearchItem item = mapMultiSearchResponseItem.result(); + if (sink.isCancelled()) { + break; + } + for (Hit hit : item.hits().hits()) { + if (sink.isCancelled()) { + break; + } + + Map hitMap = hit.source(); + if (ObjectUtils.isEmpty(hitMap.get("id"))) { + hitMap.put("id", hit.id()); + } + sink.next(mapper.apply(hitMap)); + } + } + sink.complete(); + }); } private Mono doCount(SearchRequest request) { - return restClient.count(request); + return restClient + .execute(c -> c + .count(ct -> { + ct.index(request.index()) + .query(request.query()); + return ct; + }).count()); } - protected Mono createSearchRequest(QueryParam queryParam, String... indexes) { + protected Mono createSearchRequest(QueryParam queryParam, + String... indexes) { return indexManager .getIndexesMetadata(indexes) .collectList() @@ -622,23 +626,31 @@ public class ReactiveElasticSearchService implements ElasticSearchService, Comma .flatMap(list -> createSearchRequest(queryParam, list)); } + protected int computeTrackHits(QueryParam param) { + if (param instanceof QueryParamEntity p) { + if (p.getTotal() != null) { + return 0; + } + } + return param.isPaging() ? Integer.MAX_VALUE : 0; + } + protected Mono createSearchRequest(QueryParam queryParam, List indexes) { - SearchSourceBuilder builder = ElasticSearchConverter.convertSearchSourceBuilder(queryParam, indexes.get(0)); - return Flux.fromIterable(indexes) - .flatMap(index -> getIndexForSearch(index.getIndex())) - .collectList() - .map(indexList -> - new SearchRequest(indexList.toArray(new String[0])) - .source(builder) - .indicesOptions(indexOptions)); + return Flux + .fromIterable(indexes) + .flatMap(index -> getIndexForSearch(index.getIndex())) + .collectList() + .map(indexList -> SearchRequest.of( + builder -> { + builder.ignoreUnavailable(true); + builder.allowNoIndices(true); + int track = computeTrackHits(queryParam); + builder.trackTotalHits(TrackHits.of(b -> track <= 0 ? b.enabled(false) : b.count(track))); + builder.index(indexList); + return QueryParamTranslator.convertSearchRequestBuilder(builder, queryParam, indexes.get(0)); + })); } - protected Mono createQueryBuilder(QueryParam queryParam, String index) { - return indexManager - .getIndexMetadata(index) - .map(metadata -> QueryParamTranslator.createQueryBuilder(queryParam, metadata)) - .switchIfEmpty(Mono.fromSupplier(() -> QueryParamTranslator.createQueryBuilder(queryParam, null))); - } } diff --git a/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/service/reactive/ReactiveElasticsearchClient.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/service/reactive/ReactiveElasticsearchClient.java new file mode 100755 index 00000000..41f73a3c --- /dev/null +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/service/reactive/ReactiveElasticsearchClient.java @@ -0,0 +1,42 @@ +package org.jetlinks.community.elastic.search.service.reactive; + +import co.elastic.clients.elasticsearch.ElasticsearchAsyncClient; +import co.elastic.clients.elasticsearch.ElasticsearchClient; +import co.elastic.clients.transport.Version; +import lombok.SneakyThrows; +import reactor.core.publisher.Mono; + +import java.util.concurrent.CompletableFuture; +import java.util.function.Function; + +public interface ReactiveElasticsearchClient { + + Version serverVersion(); + + Mono execute(ElasticsearchClientCallback callback); + + Mono executeAsync(ElasticsearchAsyncClientCallback callback); + + + interface ElasticsearchAsyncClientCallback extends Function> { + + @Override + @SneakyThrows + default CompletableFuture apply(ElasticsearchAsyncClient client) { + return execute(client); + } + + CompletableFuture execute(ElasticsearchAsyncClient client) throws Exception; + } + + interface ElasticsearchClientCallback extends Function { + + @Override + @SneakyThrows + default T apply(ElasticsearchClient client) { + return execute(client); + } + + T execute(ElasticsearchClient client) throws Exception; + } +} diff --git a/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/service/reactive/ScrollingFlux.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/service/reactive/ScrollingFlux.java new file mode 100644 index 00000000..96f90237 --- /dev/null +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/service/reactive/ScrollingFlux.java @@ -0,0 +1,230 @@ +package org.jetlinks.community.elastic.search.service.reactive; + +import co.elastic.clients.elasticsearch._types.Time; +import co.elastic.clients.elasticsearch.core.ScrollResponse; +import co.elastic.clients.elasticsearch.core.SearchRequest; +import co.elastic.clients.elasticsearch.core.SearchResponse; +import co.elastic.clients.elasticsearch.core.search.Hit; +import co.elastic.clients.elasticsearch.core.search.HitsMetadata; +import lombok.RequiredArgsConstructor; +import org.apache.commons.collections4.CollectionUtils; +import org.reactivestreams.Subscription; +import reactor.core.CoreSubscriber; +import reactor.core.Disposable; +import reactor.core.Disposables; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Operators; +import reactor.util.concurrent.Queues; + +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; +import java.util.concurrent.atomic.AtomicLongFieldUpdater; +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; +import java.util.function.Consumer; +import java.util.function.Function; + +public class ScrollingFlux extends Flux { + + final Function, T> mapper; + final ReactiveElasticsearchClient client; + final Time timeout; + final Consumer request; + + public ScrollingFlux(Function, T> mapper, + ReactiveElasticsearchClient client, + Time timeout, + Consumer request) { + this.mapper = mapper; + this.client = client; + this.request = request; + this.timeout = timeout; + } + + + @Override + public void subscribe(CoreSubscriber actual) { + actual.onSubscribe(new ScrollingFluxSubscriber<>(this, actual)); + } + + @RequiredArgsConstructor + static class ScrollingFluxSubscriber implements Subscription { + static final Disposable COMPLETED = Disposables.disposed(); + + final ScrollingFlux parent; + final CoreSubscriber actual; + final Queue queue = Queues.unboundedMultiproducer().get(); + + @SuppressWarnings("all") + static final AtomicIntegerFieldUpdater WIP = + AtomicIntegerFieldUpdater.newUpdater(ScrollingFluxSubscriber.class, "wip"); + volatile int wip; + + @SuppressWarnings("all") + static final AtomicLongFieldUpdater REQUESTED = + AtomicLongFieldUpdater.newUpdater(ScrollingFluxSubscriber.class, "requested"); + + volatile long requested; + + volatile String scrollId; + + volatile boolean cancelled; + + @SuppressWarnings("all") + static final AtomicReferenceFieldUpdater + FETCHING = AtomicReferenceFieldUpdater.newUpdater(ScrollingFluxSubscriber.class, Disposable.class, "fetching"); + + volatile Disposable fetching; + + Disposable doRequest() { + if (scrollId != null) { + return parent + .client + .execute(elastic -> elastic + .scroll(scroll -> { + scroll.scroll(parent.timeout) + .scrollId(scrollId); + return scroll; + }, Map.class)) + .subscribe(this::handleResponse, + this::onError); + } + return parent + .client + .execute(elastic -> elastic + .search(scroll -> { + scroll.scroll(parent.timeout); + parent.request.accept(scroll); + return scroll; + }, Map.class)) + .subscribe(this::handleResponse, + this::onError); + } + + void onError(Throwable error) { + actual.onError(error); + + } + + void handleResponse(SearchResponse response) { + this.scrollId = response.scrollId(); + handleResponse(response.hits()); + } + + void handleResponse(ScrollResponse response) { + this.scrollId = response.scrollId(); + handleResponse(response.hits()); + } + + @SuppressWarnings("all") + void handleResponse(HitsMetadata hits) { + try { + List> hitList = hits.hits(); + if (CollectionUtils.isEmpty(hitList)) { + FETCHING.set(this, COMPLETED); + drain(); + } else { + for (Hit hit : hitList) { + Map map = hit.source(); + if (map.get("id") == null) { + map.put("id", hit.id()); + } + + T res = parent.mapper.apply(map); + if (res != null) { + if (!queue.offer(res)) { + Operators.onDiscard(res, actual.currentContext()); + } + } + } + FETCHING.set(this, null); + drain(); + } + } catch (Throwable error) { + onError(error); + } + + + } + + void fetch() { + if (FETCHING.get(this) != null && WIP.getAndIncrement(this) == 0) { + drain(); + return; + } + Disposable.Swap swap = Disposables.swap(); + if (FETCHING.compareAndSet(this, null, swap)) { + swap.update(doRequest()); + } else if (WIP.getAndIncrement(this) == 0){ + drain(); + } + } + + void drain() { + if (WIP.getAndIncrement(this) != 0) { + return; + } + + int missed = 1; + + do { + long r = requested; + long count = 0; + + while (count != r) { + if (cancelled) { + return; + } + T data = queue.poll(); + if (data == null) { + //complete + if (FETCHING.get(this) == COMPLETED) { + actual.onComplete(); + return; + } + break; + } + actual.onNext(data); + count++; + } + + if (cancelled) { + return; + } + if (count != 0L) { + if (r != Long.MAX_VALUE) { + Operators.produced(REQUESTED, this, count); + } + } + + missed = WIP.addAndGet(this, -missed); + + } while (missed != 0); + if (queue.isEmpty() && FETCHING.get(this) == null) { + fetch(); + } else if (queue.isEmpty() && FETCHING.get(this) == COMPLETED) { + actual.onComplete(); + } + + } + + + @Override + public void request(long n) { + if (Operators.validate(n)) { + Operators.addCap(REQUESTED, this, n); + fetch(); + } + } + + @Override + public void cancel() { + cancelled = true; + Disposable fetching = FETCHING.getAndSet(this, COMPLETED); + if (fetching != null) { + fetching.dispose(); + } + } + } +} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchColumnModeDDLOperations.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchColumnModeDDLOperations.java similarity index 97% rename from jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchColumnModeDDLOperations.java rename to jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchColumnModeDDLOperations.java index f55d369b..ba82742d 100644 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchColumnModeDDLOperations.java +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchColumnModeDDLOperations.java @@ -6,7 +6,6 @@ import org.jetlinks.community.elastic.search.index.ElasticSearchIndexManager; import org.jetlinks.community.things.data.operations.ColumnModeDDLOperationsBase; import org.jetlinks.community.things.data.operations.DataSettings; import org.jetlinks.community.things.data.operations.MetricBuilder; -import org.jetlinks.core.things.ThingMetadata; import reactor.core.publisher.Mono; import java.util.List; diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchColumnModeQueryOperations.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchColumnModeQueryOperations.java similarity index 87% rename from jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchColumnModeQueryOperations.java rename to jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchColumnModeQueryOperations.java index b31e0348..a8715405 100644 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchColumnModeQueryOperations.java +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchColumnModeQueryOperations.java @@ -3,6 +3,7 @@ package org.jetlinks.community.elastic.search.things; import org.hswebframework.ezorm.core.dsl.Query; import org.hswebframework.web.api.crud.entity.PagerResult; import org.hswebframework.web.api.crud.entity.QueryParamEntity; +import org.jetlinks.core.metadata.EventMetadata; import org.jetlinks.core.things.ThingsRegistry; import org.jetlinks.community.elastic.search.service.AggregationService; import org.jetlinks.community.elastic.search.service.ElasticSearchService; @@ -19,10 +20,7 @@ import org.joda.time.format.DateTimeFormat; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -import java.util.Comparator; -import java.util.Date; -import java.util.HashMap; -import java.util.Map; +import java.util.*; import java.util.function.Function; class ElasticSearchColumnModeQueryOperations extends ColumnModeQueryOperationsBase { @@ -51,7 +49,7 @@ class ElasticSearchColumnModeQueryOperations extends ColumnModeQueryOperationsBa query.getParam(), data -> { long ts = CastUtils.castNumber(data.getOrDefault("timestamp", 0L)).longValue(); - data.put("timestamp",ts); + data.put("timestamp", ts); return TimeSeriesData.of(ts, data); }); } @@ -65,20 +63,27 @@ class ElasticSearchColumnModeQueryOperations extends ColumnModeQueryOperationsBa query.getParam(), data -> { long ts = CastUtils.castNumber(data.getOrDefault("timestamp", 0L)).longValue(); - data.put("timestamp",ts); + data.put("timestamp", ts); return mapper.apply(TimeSeriesData.of(ts, data)); }); } @Override protected Flux doAggregation(String metric, AggregationRequest request, AggregationContext context) { + return doAggregation0(aggregationService,metric,request,context); + } + + static Flux doAggregation0(AggregationService service, + String metric, + AggregationRequest request, + AggregationContext context) { org.joda.time.format.DateTimeFormatter formatter = DateTimeFormat.forPattern(request.getFormat()); PropertyAggregation[] properties = context.getProperties(); return AggregationQueryParam .of() .as(param -> { for (PropertyAggregation property : properties) { - param.agg(property.getProperty(), property.getAlias(), property.getAgg(),property.getDefaultValue()); + param.agg(property.getProperty(), property.getAlias(), property.getAgg(), property.getDefaultValue()); } return param; }) @@ -92,7 +97,7 @@ class ElasticSearchColumnModeQueryOperations extends ColumnModeQueryOperationsBa .from(request.getFrom()) .to(request.getTo()) .filter(request.getFilter()) - .execute(param -> aggregationService.aggregation(metric, param)) + .execute(param -> service.aggregation(metric, param)) .map(AggregationData::of) .groupBy(agg -> agg.getString("time", ""), Integer.MAX_VALUE) .flatMap(group -> group @@ -101,11 +106,11 @@ class ElasticSearchColumnModeQueryOperations extends ColumnModeQueryOperationsBa newMap.put("time", data.get("time").orElse(null)); for (PropertyAggregation property : properties) { Object val; - if(property.getAgg() ==Aggregation.FIRST || property.getAgg()==Aggregation.TOP){ + if (property.getAgg() == Aggregation.FIRST || property.getAgg() == Aggregation.TOP) { val = data .get(property.getProperty()) .orElse(null); - }else { + } else { val = data .get(property.getAlias()) .orElse(null); diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchColumnModeSaveOperations.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchColumnModeSaveOperations.java similarity index 100% rename from jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchColumnModeSaveOperations.java rename to jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchColumnModeSaveOperations.java diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchColumnModeStrategy.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchColumnModeStrategy.java similarity index 94% rename from jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchColumnModeStrategy.java rename to jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchColumnModeStrategy.java index db83c929..f2881719 100644 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchColumnModeStrategy.java +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchColumnModeStrategy.java @@ -1,12 +1,15 @@ package org.jetlinks.community.elastic.search.things; import lombok.AllArgsConstructor; +import org.jetlinks.core.metadata.Feature; +import org.jetlinks.core.metadata.MetadataFeature; import org.jetlinks.core.things.ThingsRegistry; import org.jetlinks.community.elastic.search.index.ElasticSearchIndexManager; import org.jetlinks.community.elastic.search.service.AggregationService; import org.jetlinks.community.elastic.search.service.ElasticSearchService; import org.jetlinks.community.things.data.AbstractThingDataRepositoryStrategy; import org.jetlinks.community.things.data.operations.*; +import reactor.core.publisher.Flux; @AllArgsConstructor public class ElasticSearchColumnModeStrategy extends AbstractThingDataRepositoryStrategy { diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchRowModeDDLOperations.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchRowModeDDLOperations.java similarity index 94% rename from jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchRowModeDDLOperations.java rename to jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchRowModeDDLOperations.java index e06d3ecd..9762e61a 100644 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchRowModeDDLOperations.java +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchRowModeDDLOperations.java @@ -1,6 +1,8 @@ package org.jetlinks.community.elastic.search.things; import org.jetlinks.core.metadata.PropertyMetadata; +import org.jetlinks.core.metadata.SimplePropertyMetadata; +import org.jetlinks.core.metadata.types.*; import org.jetlinks.community.elastic.search.index.DefaultElasticSearchIndexMetadata; import org.jetlinks.community.elastic.search.index.ElasticSearchIndexManager; import org.jetlinks.community.things.data.operations.DataSettings; @@ -24,6 +26,11 @@ class ElasticSearchRowModeDDLOperations extends RowModeDDLOperationsBase { this.indexManager = indexManager; } + @Override + protected boolean isOnlySupportsOneObjectOrArrayProperty() { + return true; + } + @Override protected Mono register(MetricType metricType, String metric, List properties) { return indexManager @@ -35,9 +42,4 @@ class ElasticSearchRowModeDDLOperations extends RowModeDDLOperationsBase { return indexManager .putIndex(new DefaultElasticSearchIndexMetadata(metric, properties)); } - - @Override - protected boolean isOnlySupportsOneObjectOrArrayProperty() { - return true; - } } diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchRowModeQueryOperations.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchRowModeQueryOperations.java similarity index 92% rename from jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchRowModeQueryOperations.java rename to jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchRowModeQueryOperations.java index 2b43150e..47f85dd4 100644 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchRowModeQueryOperations.java +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchRowModeQueryOperations.java @@ -3,6 +3,7 @@ package org.jetlinks.community.elastic.search.things; import org.hswebframework.ezorm.core.dsl.Query; import org.hswebframework.web.api.crud.entity.PagerResult; import org.hswebframework.web.api.crud.entity.QueryParamEntity; +import org.jetlinks.core.metadata.EventMetadata; import org.jetlinks.core.metadata.PropertyMetadata; import org.jetlinks.core.things.ThingMetadata; import org.jetlinks.core.things.ThingsRegistry; @@ -75,7 +76,9 @@ class ElasticSearchRowModeQueryOperations extends RowModeQueryOperationsBase { @Nonnull Map properties) { QueryParamEntity param = query.getParam(); //不分页或者每页数量大于1000则回退到普通方式查询 - if (!param.isPaging() || param.getPageSize() >= 1000) { + if (!param.isPaging() || param.getPageSize() >= 1000 || !settings + .getProperty() + .isQueryPropertiesAggregation()) { return super.queryEachProperty(metric, query, metadata, properties); } @@ -115,9 +118,10 @@ class ElasticSearchRowModeQueryOperations extends RowModeQueryOperationsBase { PropertyAggregation[] properties = context.getProperties(); //只聚合一个属性时 if (properties.length == 1) { + Aggregation aggregation = properties[0].getAgg(); return AggregationQueryParam .of() - .agg(ThingsDataConstants.COLUMN_PROPERTY_NUMBER_VALUE, properties[0].getAlias(), properties[0].getAgg(), properties[0].getDefaultValue()) + .agg(getPropertyColumn(aggregation), properties[0].getAlias(), aggregation, properties[0].getDefaultValue()) .as(param -> { if (request.getInterval() == null) { return param; @@ -143,7 +147,10 @@ class ElasticSearchRowModeQueryOperations extends RowModeQueryOperationsBase { .of() .as(param -> { Arrays.stream(properties) - .forEach(agg -> param.agg(ThingsDataConstants.COLUMN_PROPERTY_NUMBER_VALUE, "value_" + agg.getAlias(), agg.getAgg(),agg.getDefaultValue())); + .forEach(agg -> { + Aggregation aggregation = agg.getAgg(); + param.agg(getPropertyColumn(aggregation), "value_" + agg.getAlias(), aggregation, agg.getDefaultValue()); + }); return param; }) .as(param -> { @@ -183,7 +190,7 @@ class ElasticSearchRowModeQueryOperations extends RowModeQueryOperationsBase { data.put("_time", agg.get("_time").orElse(time)); data.put("time", time); aliasProperty.forEach((alias, prp) -> { - if (prp.getAgg() == Aggregation.FIRST || prp.getAgg() == Aggregation.TOP) { + if (prp.getAgg() == Aggregation.FIRST || prp.getAgg() == Aggregation.TOP || prp.getAgg() == Aggregation.LAST) { data.putIfAbsent(alias, agg .get(ThingsDataConstants.COLUMN_PROPERTY_NUMBER_VALUE) .orElse(agg.get("value").orElse(null))); @@ -236,4 +243,11 @@ class ElasticSearchRowModeQueryOperations extends RowModeQueryOperationsBase { .take(request.getLimit()) ; } + + private static String getPropertyColumn(Aggregation aggregation) { + if (aggregation.needNumberValue()) { + return ThingsDataConstants.COLUMN_PROPERTY_NUMBER_VALUE; + } + return ThingsDataConstants.COLUMN_PROPERTY_VALUE; + } } diff --git a/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchRowModeSaveOperations.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchRowModeSaveOperations.java new file mode 100644 index 00000000..d7fbe3a7 --- /dev/null +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchRowModeSaveOperations.java @@ -0,0 +1,66 @@ +package org.jetlinks.community.elastic.search.things; + +import org.jetlinks.core.metadata.DataType; +import org.jetlinks.core.metadata.PropertyMetadata; +import org.jetlinks.core.metadata.types.ArrayType; +import org.jetlinks.core.metadata.types.ObjectType; +import org.jetlinks.core.things.ThingsRegistry; +import org.jetlinks.community.elastic.search.service.ElasticSearchService; +import org.jetlinks.community.things.data.operations.DataSettings; +import org.jetlinks.community.things.data.operations.MetricBuilder; +import org.jetlinks.community.things.data.operations.RowModeSaveOperationsBase; +import org.jetlinks.community.timeseries.TimeSeriesData; +import org.jetlinks.community.utils.ObjectMappers; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.util.Map; + +import static org.jetlinks.community.things.data.ThingsDataConstants.*; + +class ElasticSearchRowModeSaveOperations extends RowModeSaveOperationsBase { + + private final ElasticSearchService searchService; + + public ElasticSearchRowModeSaveOperations(ThingsRegistry registry, + MetricBuilder metricBuilder, + DataSettings settings, + ElasticSearchService searchService) { + super(registry, metricBuilder, settings); + this.searchService = searchService; + } + + @Override + protected Mono doSave(String metric, TimeSeriesData data) { + return searchService.commit(metric, data.getData()); + } + + @Override + protected Mono doSave(String metric, Flux data) { + return searchService.save(metric, data.map(TimeSeriesData::getData)); + } + + @Override + protected void fillRowPropertyValue(Map target, PropertyMetadata property, Object value) { + DataType type = property.getValueType(); + if (type instanceof ArrayType && value != null) { + if (propertyIsJsonStringStorage(property)) { + String convertedValue = value instanceof String + ? String.valueOf(value) + : ObjectMappers.toJsonString(value); + target.put(COLUMN_PROPERTY_VALUE, convertedValue); + return; + } + + ArrayType objectType = (ArrayType) type; + DataType elementType = objectType.getElementType(); + if (elementType instanceof ObjectType) { + Object val = objectType.convert(value); + target.put(COLUMN_PROPERTY_OBJECT_VALUE, val); + target.put(COLUMN_PROPERTY_VALUE, ObjectMappers.toJsonString(val)); + return; + } + } + super.fillRowPropertyValue(target, property, value); + } +} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchRowModeStrategy.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchRowModeStrategy.java similarity index 94% rename from jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchRowModeStrategy.java rename to jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchRowModeStrategy.java index 36366ac1..f48655be 100644 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchRowModeStrategy.java +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchRowModeStrategy.java @@ -1,12 +1,15 @@ package org.jetlinks.community.elastic.search.things; import lombok.AllArgsConstructor; +import org.jetlinks.core.metadata.Feature; +import org.jetlinks.core.metadata.MetadataFeature; import org.jetlinks.core.things.ThingsRegistry; import org.jetlinks.community.elastic.search.index.ElasticSearchIndexManager; import org.jetlinks.community.elastic.search.service.AggregationService; import org.jetlinks.community.elastic.search.service.ElasticSearchService; import org.jetlinks.community.things.data.AbstractThingDataRepositoryStrategy; import org.jetlinks.community.things.data.operations.*; +import reactor.core.publisher.Flux; @AllArgsConstructor public class ElasticSearchRowModeStrategy extends AbstractThingDataRepositoryStrategy { diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/timeseries/ElasticSearchTimeSeriesManager.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/timeseries/ElasticSearchTimeSeriesManager.java old mode 100644 new mode 100755 similarity index 89% rename from jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/timeseries/ElasticSearchTimeSeriesManager.java rename to jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/timeseries/ElasticSearchTimeSeriesManager.java index a3f5d9d6..5f16f479 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/timeseries/ElasticSearchTimeSeriesManager.java +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/timeseries/ElasticSearchTimeSeriesManager.java @@ -1,6 +1,9 @@ package org.jetlinks.community.elastic.search.timeseries; import lombok.extern.slf4j.Slf4j; +import org.jetlinks.core.cache.Caches; +import org.jetlinks.core.metadata.SimplePropertyMetadata; +import org.jetlinks.core.metadata.types.DateTimeType; import org.jetlinks.community.elastic.search.index.DefaultElasticSearchIndexMetadata; import org.jetlinks.community.elastic.search.index.ElasticSearchIndexManager; import org.jetlinks.community.elastic.search.service.AggregationService; @@ -9,14 +12,10 @@ import org.jetlinks.community.timeseries.TimeSeriesManager; import org.jetlinks.community.timeseries.TimeSeriesMetadata; import org.jetlinks.community.timeseries.TimeSeriesMetric; import org.jetlinks.community.timeseries.TimeSeriesService; -import org.jetlinks.core.metadata.SimplePropertyMetadata; -import org.jetlinks.core.metadata.types.DateTimeType; -import org.springframework.stereotype.Service; import reactor.core.publisher.Mono; import java.util.Arrays; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; /** * @author bsetfeng @@ -27,7 +26,7 @@ import java.util.concurrent.ConcurrentHashMap; public class ElasticSearchTimeSeriesManager implements TimeSeriesManager { - private final Map serviceMap = new ConcurrentHashMap<>(16); + private final Map serviceMap = Caches.newCache(); protected final ElasticSearchIndexManager indexManager; @@ -74,7 +73,8 @@ public class ElasticSearchTimeSeriesManager implements TimeSeriesManager { SimplePropertyMetadata timestamp = new SimplePropertyMetadata(); timestamp.setId("timestamp"); timestamp.setValueType(new DateTimeType()); - return indexManager.putIndex(new DefaultElasticSearchIndexMetadata(metadata.getMetric().getId(), metadata.getProperties()) + return indexManager + .putIndex(new DefaultElasticSearchIndexMetadata(metadata.getMetric().getId(), metadata.getProperties()) .addProperty(timestamp)); } diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/timeseries/ElasticSearchTimeSeriesService.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/timeseries/ElasticSearchTimeSeriesService.java old mode 100644 new mode 100755 similarity index 62% rename from jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/timeseries/ElasticSearchTimeSeriesService.java rename to jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/timeseries/ElasticSearchTimeSeriesService.java index 0a153ff2..837e6075 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/timeseries/ElasticSearchTimeSeriesService.java +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/timeseries/ElasticSearchTimeSeriesService.java @@ -1,22 +1,25 @@ package org.jetlinks.community.elastic.search.timeseries; import lombok.AllArgsConstructor; +import lombok.Generated; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.hswebframework.ezorm.core.param.QueryParam; import org.hswebframework.web.api.crud.entity.PagerResult; +import org.hswebframework.web.id.IDGenerator; import org.jetlinks.community.elastic.search.service.AggregationService; import org.jetlinks.community.elastic.search.service.ElasticSearchService; import org.jetlinks.community.timeseries.TimeSeriesData; import org.jetlinks.community.timeseries.TimeSeriesService; import org.jetlinks.community.timeseries.query.AggregationData; import org.jetlinks.community.timeseries.query.AggregationQueryParam; -import org.jetlinks.core.metadata.types.DateTimeType; +import org.jetlinks.reactor.ql.utils.CastUtils; import org.reactivestreams.Publisher; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import java.util.Collection; +import java.util.HashMap; import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; @@ -31,11 +34,11 @@ public class ElasticSearchTimeSeriesService implements TimeSeriesService { private final AggregationService aggregationService; - static DateTimeType timeType = DateTimeType.GLOBAL; - @Override public Flux query(QueryParam queryParam) { - return elasticSearchService.query(index, applySort(queryParam), map -> TimeSeriesData.of(timeType.convert(map.get("timestamp")), map)); + return elasticSearchService + .query(index, applySort(queryParam), + this::convertTimeSeriesData); } @Override @@ -43,7 +46,7 @@ public class ElasticSearchTimeSeriesService implements TimeSeriesService { return elasticSearchService.multiQuery( index, query.stream().peek(this::applySort).collect(Collectors.toList()), - map -> TimeSeriesData.of(timeType.convert(map.get("timestamp")), map)); + this::convertTimeSeriesData); } @Override @@ -55,12 +58,17 @@ public class ElasticSearchTimeSeriesService implements TimeSeriesService { @Override public Mono> queryPager(QueryParam queryParam) { - return elasticSearchService.queryPager(index, applySort(queryParam), map -> TimeSeriesData.of(timeType.convert(map.get("timestamp")), map)); + return elasticSearchService + .queryPager(index, applySort(queryParam) + , this::convertTimeSeriesData); } @Override public Mono> queryPager(QueryParam queryParam, Function mapper) { - return elasticSearchService.queryPager(index, applySort(queryParam), map -> mapper.apply(TimeSeriesData.of(timeType.convert(map.get("timestamp")), map))); + return elasticSearchService.queryPager( + index, + applySort(queryParam), + map -> mapper.apply(this.convertTimeSeriesData(map))); } @Override @@ -71,7 +79,9 @@ public class ElasticSearchTimeSeriesService implements TimeSeriesService { log.error(err.getMessage(), err); return Mono.empty(); }) - .map(AggregationData::of); + .map(AggregationData::of) +// .as(flux -> queryParam.getLimit() > 0 ? flux.take(queryParam.getLimit()) : flux) + ; } @@ -86,26 +96,36 @@ public class ElasticSearchTimeSeriesService implements TimeSeriesService { @Override public Mono commit(Publisher data) { return Flux.from(data) - .flatMap(this::commit) - .then(); + .flatMap(this::commit) + .then(); } @Override public Mono commit(TimeSeriesData data) { - Map mapData = data.getData(); + Map mapData = new HashMap<>(data.getData()); mapData.put("timestamp", data.getTimestamp()); + mapData.computeIfAbsent("id", ignore -> IDGenerator.RANDOM.generate()); return elasticSearchService.commit(index[0], mapData); } @Override public Mono save(Publisher dateList) { - return elasticSearchService.save(index[0], - Flux.from(dateList) - .map(data -> { - Map mapData = data.getData(); - mapData.put("timestamp", data.getTimestamp()); - return mapData; - })); + return elasticSearchService + .save(index[0], + Flux.from(dateList) + .map(data -> { + Map mapData = new HashMap<>(data.getData()); + mapData.put("timestamp", data.getTimestamp()); + mapData.computeIfAbsent("id", ignore -> IDGenerator.RANDOM.generate()); + return mapData; + })); + } + + public TimeSeriesData convertTimeSeriesData(Map data) { + return TimeSeriesData + .of(CastUtils + .castNumber(data.getOrDefault("timestamp", 0)) + .longValue(), data); } } diff --git a/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/trace/TraceInstrumentation.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/trace/TraceInstrumentation.java new file mode 100644 index 00000000..d402de9e --- /dev/null +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/trace/TraceInstrumentation.java @@ -0,0 +1,138 @@ +package org.jetlinks.community.elastic.search.trace; + +import co.elastic.clients.elasticsearch.core.SearchRequest; +import co.elastic.clients.transport.Endpoint; +import co.elastic.clients.transport.TransportOptions; +import co.elastic.clients.transport.http.TransportHttpClient; +import co.elastic.clients.transport.instrumentation.Instrumentation; +import co.elastic.clients.transport.instrumentation.NoopInstrumentation; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.context.Scope; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.jetlinks.core.lang.SeparatedCharSequence; +import org.jetlinks.core.lang.SharedPathString; +import org.jetlinks.core.message.codec.http.HttpUtils; +import org.jetlinks.core.trace.TraceHolder; + +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +@Slf4j +public class TraceInstrumentation implements Instrumentation { + private static final Set SEARCH_ENDPOINTS = new HashSet<>(Arrays.asList( + "render_search_template", + "terms_enum", + "msearch_template", + "eql.search", + "msearch", + "search_template", + "async_search.submit", + "search" + )); + + private static final AttributeKey ATTR_HTTP_REQUEST_METHOD = AttributeKey.stringKey("http.request.method"); + private static final AttributeKey ATTR_REQUEST_URI = AttributeKey.stringKey("http.request.uri"); + private static final AttributeKey ATTR_REQUEST_BODY = AttributeKey.stringKey("http.request.body"); + + private final static SharedPathString spanNameTemplate = + SharedPathString.of("/_elastic/*"); + + @Override + public Context newContext(TRequest tRequest, Endpoint endpoint) { + String endpointId = endpoint.id(); + if (endpointId.startsWith("es/")) { + endpointId = endpointId.substring(3); + } + SeparatedCharSequence spanName = createSpanName(spanNameTemplate.replace(2, endpointId), tRequest); + + if (TraceHolder.isDisabled(spanName)) { + return NoopInstrumentation.INSTANCE.newContext(tRequest, endpoint); + } + Span span = TraceHolder + .telemetry() + .getTracer(TraceHolder.appName()) + .spanBuilder(spanName.toString()) + .setParent(io.opentelemetry.context.Context.current()) + .startSpan(); + + return new TraceContext(span, endpointId); + } + + private SeparatedCharSequence createSpanName(SeparatedCharSequence prefix,Object request){ + if(request instanceof SearchRequest req){ + return prefix.append(String.join(",", req.index())); + } + return prefix; + } + + @RequiredArgsConstructor + static class TraceContext implements Context { + final Span span; + final String endpointId; + + String pathAndQuery; + + @Override + @SuppressWarnings("all") + public ThreadScope makeCurrent() { + Scope scope = span.makeCurrent(); + return scope::close; + } + + @Override + public void beforeSendingHttpRequest(TransportHttpClient.Request httpRequest, TransportOptions options) { + + pathAndQuery = HttpUtils.appendUrlParameter(httpRequest.path(), httpRequest.queryParams()); + + span.setAttribute(ATTR_HTTP_REQUEST_METHOD, httpRequest.method()); + + Iterable body = httpRequest.body(); + if (span.isRecording() && body != null && SEARCH_ENDPOINTS.contains(endpointId)) { + StringBuilder sb = new StringBuilder(); + for (ByteBuffer buf : body) { + buf.mark(); + sb.append(StandardCharsets.UTF_8.decode(buf)); + buf.reset(); + } + span.setAttribute(ATTR_REQUEST_BODY, sb.toString()); + } + } + + @Override + public void afterReceivingHttpResponse(TransportHttpClient.Response httpResponse) { + try { + if (span.isRecording()) { + URI uri = httpResponse.node().uri(); + String fullUrl = uri.resolve(pathAndQuery).toString(); + span.setAttribute(ATTR_REQUEST_URI, fullUrl); + } + } catch (RuntimeException e) { + log.debug("Failed capturing response information for the OpenTelemetry span.", e); + // ignore + } + } + + @Override + public void afterDecodingApiResponse(TResponse apiResponse) { + + } + + @Override + public void recordException(Throwable thr) { + span.setStatus(StatusCode.ERROR, thr.getMessage()); + span.recordException(thr); + } + + @Override + public void close() { + span.end(); + } + } +} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/utils/ElasticSearchConverter.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/utils/ElasticSearchConverter.java old mode 100644 new mode 100755 similarity index 85% rename from jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/utils/ElasticSearchConverter.java rename to jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/utils/ElasticSearchConverter.java index 8c437b2c..b29d1d8e --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/utils/ElasticSearchConverter.java +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/utils/ElasticSearchConverter.java @@ -1,9 +1,6 @@ package org.jetlinks.community.elastic.search.utils; import com.google.common.collect.Collections2; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.hswebframework.ezorm.core.param.QueryParam; -import org.jetlinks.community.elastic.search.index.ElasticSearchIndexMetadata; import org.jetlinks.core.metadata.Converter; import org.jetlinks.core.metadata.DataType; import org.jetlinks.core.metadata.PropertyMetadata; @@ -14,10 +11,6 @@ import java.util.*; public class ElasticSearchConverter { - public static SearchSourceBuilder convertSearchSourceBuilder(QueryParam queryParam, ElasticSearchIndexMetadata metadata) { - return QueryParamTranslator.convertSearchSourceBuilder(queryParam, metadata); - } - public static Map convertDataToElastic(Map data, List properties) { Map newValue = new HashMap<>(data); @@ -67,9 +60,9 @@ public class ElasticSearchConverter { //处理地理位置类型 if (type instanceof GeoType) { newData.put(property.getId(), ((GeoType) type).convertToMap(val)); - }else if (type instanceof GeoShapeType) { - newData.put(property.getId(),GeoShape.of(val).toMap()); - } else if (type instanceof DateTimeType) { + } else if (type instanceof GeoShapeType) { + newData.put(property.getId(), GeoShape.of(val).toMap()); + } else if (type instanceof DateTimeType) { Date date = ((DateTimeType) type).convert(val); newData.put(property.getId(), date); } else if (type instanceof ObjectType) { @@ -85,4 +78,4 @@ public class ElasticSearchConverter { } return newData; } -} \ No newline at end of file +} diff --git a/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/utils/QueryParamTranslator.java b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/utils/QueryParamTranslator.java new file mode 100755 index 00000000..f23f3ccf --- /dev/null +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/java/org/jetlinks/community/elastic/search/utils/QueryParamTranslator.java @@ -0,0 +1,295 @@ +package org.jetlinks.community.elastic.search.utils; + +import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery; +import co.elastic.clients.elasticsearch._types.query_dsl.ChildScoreMode; +import co.elastic.clients.elasticsearch._types.query_dsl.NestedQuery; +import co.elastic.clients.elasticsearch._types.query_dsl.Query; +import co.elastic.clients.elasticsearch.core.SearchRequest; +import co.elastic.clients.elasticsearch.core.msearch.MultisearchBody; +import lombok.Getter; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import org.hswebframework.ezorm.core.param.QueryParam; +import org.hswebframework.ezorm.core.param.Sort; +import org.hswebframework.ezorm.core.param.Term; +import org.hswebframework.ezorm.core.param.TermType; +import org.jetlinks.core.metadata.DataType; +import org.jetlinks.core.metadata.PropertyMetadata; +import org.jetlinks.core.metadata.types.ArrayType; +import org.jetlinks.core.metadata.types.ObjectType; +import org.jetlinks.community.elastic.search.enums.ElasticSearchTermType; +import org.jetlinks.community.elastic.search.enums.ElasticSearchTermTypes; +import org.jetlinks.community.elastic.search.index.ElasticSearchIndexMetadata; +import org.jetlinks.community.things.utils.ThingsDatabaseUtils; +import org.springframework.util.CollectionUtils; +import org.springframework.util.ObjectUtils; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.function.BiFunction; +import java.util.function.Consumer; + +/** + * @author zhouhao + * @author bestfeng + * @since 1.0 + **/ +@Slf4j +public class QueryParamTranslator { + + static Consumer doNotingParamConverter = (term -> { + }); + + public static Query.Builder applyQueryBuilder(Query.Builder queryBuilder, + QueryParam queryParam, + ElasticSearchIndexMetadata metadata) { + return applyQueryBuilder(queryBuilder, queryParam.getTerms(), metadata); + } + + public static Query.Builder applyQueryBuilder(Query.Builder queryBuilder, + List terms, + ElasticSearchIndexMetadata metadata) { + Consumer paramConverter = doNotingParamConverter; + if (metadata != null) { + paramConverter = term -> { + if (ObjectUtils.isEmpty(term.getColumn())) { + return; + } + ElasticSearchTermType termType = ElasticSearchTermTypes.lookupNow(term); + PropertyMetadata property = metadata.getProperty(term.getColumn()); + if (null != property) { + DataType type = property.getValueType(); + ThingsDatabaseUtils.tryConvertTermValue( + type, + term, + termType::convertTermValue); + } + }; + } + + + BiFunction nestedConverter = (a, b) -> b; + if (metadata != null) { + nestedConverter = (term, builder) -> { + if (!ObjectUtils.isEmpty(term.getColumn())) { + return nestedTermConverter(term, metadata, builder); + } + return builder; + }; + } + Consumer fParamConverter = paramConverter; + BiFunction fNestedConverter = nestedConverter; + + queryBuilder.bool(bool -> process( + terms, + fParamConverter, + fNestedConverter, bool)); + + return queryBuilder; + } + + public static NestedQuery.Builder nestedTermConverter(Term term, + ElasticSearchIndexMetadata metadata, + NestedQuery.Builder queryBuilder) { + ElasticSearchTermType termType = ElasticSearchTermTypes.lookup(term).orElse(null); + if (termType != null) { + PropertyMetadata property = metadata.getProperty(term.getColumn()); + if (null != property && (TermType.isnull.equals(termType.getId()) || TermType.notnull.equals(termType.getId()))) { + if (property.getValueType() instanceof ArrayType arrayType) { + // nested类型无法直接使用exists判断 + if (arrayType.getElementType() instanceof ObjectType) { + queryBuilder.scoreMode(ChildScoreMode.None); + } + } + if (property.getValueType() instanceof ObjectType) { + // nested类型无法直接使用exists判断 + queryBuilder.scoreMode(ChildScoreMode.None); + } + } + } + return queryBuilder; + + } + + + public static MultisearchBody.Builder convertSearchRequestBuilder( + MultisearchBody.Builder builder, + QueryParam queryParam, + ElasticSearchIndexMetadata metadata) { + + if (queryParam.isPaging()) { + builder.from(queryParam.getPageIndex() * queryParam.getPageSize()); + builder.size(queryParam.getPageSize()); + } + for (Sort sort : queryParam.getSorts()) { + builder.sort(_sort -> _sort.field(field -> field + .field(sort.getName()) + .order(sort.getOrder() + .equalsIgnoreCase("ASC") ? + co.elastic.clients.elasticsearch._types.SortOrder.Asc : co.elastic.clients.elasticsearch._types.SortOrder.Desc))); + } + + builder.query(query -> applyQueryBuilder(query, queryParam, metadata)); + + return builder; + } + + public static SearchRequest.Builder convertSearchRequestBuilder( + SearchRequest.Builder builder, + QueryParam queryParam, + ElasticSearchIndexMetadata metadata) { + + if (queryParam.isPaging()) { + builder.from(queryParam.getPageIndex() * queryParam.getPageSize()); + builder.size(queryParam.getPageSize()); + } + for (Sort sort : queryParam.getSorts()) { + builder.sort(_sort -> _sort.field(field -> field + .field(sort.getName()) + .order(sort.getOrder() + .equalsIgnoreCase("ASC") ? + co.elastic.clients.elasticsearch._types.SortOrder.Asc : co.elastic.clients.elasticsearch._types.SortOrder.Desc))); + } + + builder.query(query -> applyQueryBuilder(query, queryParam, metadata)); + + return builder; + } + + + public static BoolQuery.Builder process(List terms, + Consumer consumer, + BiFunction nestedConverter, + BoolQuery.Builder queryBuilders) { + + if (CollectionUtils.isEmpty(terms)) { + return queryBuilders; + } + + for (TermGroup group : groupTerms(terms)) { + if (group.type == Term.Type.or) { + for (Term groupTerm : group.getTerms()) { + handleOr(queryBuilders, groupTerm, nestedConverter, consumer); + } + } else { + queryBuilders + .should(must -> + must.bool(bool -> { + for (Term groupTerm : group.getTerms()) { + handleAnd(bool, groupTerm, nestedConverter, consumer); + } + return bool; + })); + } + + } + return queryBuilders; + } + + private static void handleOr(BoolQuery.Builder queryBuilders, + Term term, + BiFunction nestedConverter, + Consumer consumer) { + consumer.accept(term); + if (term.getTerms().isEmpty() && term.getValue() != null) { + + queryBuilders + .should(should -> applyTerm(term, should, nestedConverter)); + + } else if (!term.getTerms().isEmpty()) { + + queryBuilders + .should(should -> should + .bool(bool -> process(term.getTerms(), consumer, nestedConverter, bool))); + } + } + + private static Query.Builder applyTerm(Term term, Query.Builder builder, BiFunction converter) { + + if (term.getColumn().contains(".")) { + String path = term.getColumn().split("[.]")[0]; + + builder.nested(n -> converter + .apply(term, n + .path(path) + .boost(1F) + .ignoreUnmapped(false) + .query(query -> ElasticSearchTermTypes + .lookup(term) + .map(type -> type.process(term, query)) + .orElse(query))) + ); + + return builder; + } + return ElasticSearchTermTypes + .lookup(term) + .map(type -> type.process(term, builder)) + .orElse(builder); + } + + private static void handleAnd(BoolQuery.Builder queryBuilders, + Term term, + BiFunction nestedConverter, + Consumer consumer) { + consumer.accept(term); + if (term.getTerms().isEmpty() && term.getValue() != null) { + + queryBuilders.must(must -> applyTerm(term, must, nestedConverter)); + + } else if (!term.getTerms().isEmpty()) { + + queryBuilders.must(must -> must.bool(bool -> process(term.getTerms(), consumer, nestedConverter, bool))); + } + } + + + public static Set groupTerms(List terms) { + Set groups = new HashSet<>(); + TermGroup currentGroup = null; + + for (int i = 0; i < terms.size(); i++) { + Term currentTerm = terms.get(i); + Term nextTerm = (i + 1 < terms.size()) ? terms.get(i + 1) : null; + + if (currentTerm.getType() == Term.Type.or) { + if (currentGroup == null || currentGroup.type == Term.Type.and) { + currentGroup = new TermGroup(Term.Type.or); + } + // 如果下一个Term为"AND"，创建一个新分组 + if (nextTerm != null && nextTerm.getType() == Term.Type.and) { + currentGroup = new TermGroup(Term.Type.and); + } + } else { + if (currentGroup == null) { + currentGroup = new TermGroup(Term.Type.and); + } + } + currentGroup.addTerm(currentTerm); + groups.add(currentGroup); + } + return groups; + } + + @Getter + @Setter + public static class TermGroup { + public TermGroup(Term.Type type) { + this.type = type; + this.terms = new ArrayList<>(); + } + + List terms; + + Term.Type type; + + public void addTerm(Term term) { + terms.add(term); + } + } + + +} \ No newline at end of file diff --git a/jetlinks-components/elasticsearch-component/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports similarity index 56% rename from jetlinks-components/elasticsearch-component/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports rename to jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports index 33c3082d..2ee6b147 100644 --- a/jetlinks-components/elasticsearch-component/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports +++ b/jetlinks-components/elasticsearch-component/elasticsearch-core/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports @@ -1,2 +1,3 @@ +org.jetlinks.community.elastic.search.configuration.ElasticSearchClientConfiguration org.jetlinks.community.elastic.search.configuration.ElasticSearchConfiguration org.jetlinks.community.elastic.search.configuration.ElasticSearchThingDataConfiguration \ No newline at end of file diff --git a/jetlinks-components/elasticsearch-component/pom.xml b/jetlinks-components/elasticsearch-component/pom.xml old mode 100644 new mode 100755 index 1a3a9a41..5096eb97 --- a/jetlinks-components/elasticsearch-component/pom.xml +++ b/jetlinks-components/elasticsearch-component/pom.xml @@ -5,78 +5,20 @@ jetlinks-components org.jetlinks.community - 2.3.0-SNAPSHOT + 2.10.0-SNAPSHOT ../pom.xml 4.0.0 + pom + elasticsearch-component - + + elasticsearch-core + elasticsearch-8x + elasticsearch-7x + - - org.elasticsearch.plugin - transport-netty4-client - - - - org.apache.logging.log4j - log4j-core - - - - org.slf4j - log4j-over-slf4j - - - - io.projectreactor - reactor-core - - - - org.elasticsearch.client - elasticsearch-rest-high-level-client - - - - org.springframework.data - spring-data-elasticsearch - - - - org.hswebframework.web - hsweb-commons-crud - ${hsweb.framework.version} - - - - org.hswebframework - hsweb-easy-orm-rdb - - - - org.jetlinks - jetlinks-core - ${jetlinks.version} - - - - ${project.groupId} - timeseries-component - ${project.version} - - - - io.projectreactor.netty - reactor-netty - - - - ${project.groupId} - things-component - ${project.version} - - \ No newline at end of file diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/elasticsearch/common/logging/Loggers.java b/jetlinks-components/elasticsearch-component/src/main/java/org/elasticsearch/common/logging/Loggers.java deleted file mode 100644 index c96efc06..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/elasticsearch/common/logging/Loggers.java +++ /dev/null @@ -1,148 +0,0 @@ -package org.elasticsearch.common.logging; - -import org.apache.logging.log4j.Level; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.core.Appender; -import org.elasticsearch.common.settings.Setting; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.shard.ShardId; - -import static org.elasticsearch.common.util.CollectionUtils.asArrayList; - -/** - * A set of utilities around Logging. - */ -public class Loggers { - - public static final String SPACE = " "; - - public static final Setting LOG_DEFAULT_LEVEL_SETTING = - new Setting<>("logger.level", Level.INFO.name(), Level::valueOf, Setting.Property.NodeScope); - public static final Setting.AffixSetting LOG_LEVEL_SETTING = - Setting.prefixKeySetting("logger.", (key) -> new Setting<>(key, Level.INFO.name(), Level::valueOf, Setting.Property.Dynamic, - Setting.Property.NodeScope)); - - public static Logger getLogger(Class clazz, ShardId shardId, String... prefixes) { - return getLogger(clazz, shardId.getIndex(), asArrayList(Integer.toString(shardId.id()), prefixes).toArray(new String[0])); - } - - /** - * Just like {@link #getLogger(Class, ShardId, String...)} but String loggerName instead of - * Class and no extra prefixes. - */ - public static Logger getLogger(String loggerName, ShardId shardId) { - String prefix = formatPrefix(shardId.getIndexName(), Integer.toString(shardId.id())); - return new PrefixLogger(LogManager.getLogger(loggerName), prefix); - } - - public static Logger getLogger(Class clazz, Index index, String... prefixes) { - return getLogger(clazz, asArrayList(Loggers.SPACE, index.getName(), prefixes).toArray(new String[0])); - } - - public static Logger getLogger(Class clazz, String... prefixes) { - return new PrefixLogger(LogManager.getLogger(clazz), formatPrefix(prefixes)); - } - - public static Logger getLogger(Logger parentLogger, String s) { - Logger inner = LogManager.getLogger(parentLogger.getName() + s); - if (parentLogger instanceof PrefixLogger) { - return new PrefixLogger(inner, ((PrefixLogger)parentLogger).prefix()); - } - return inner; - } - - private static String formatPrefix(String... prefixes) { - String prefix = null; - if (prefixes != null && prefixes.length > 0) { - StringBuilder sb = new StringBuilder(); - for (String prefixX : prefixes) { - if (prefixX != null) { - if (prefixX.equals(SPACE)) { - sb.append(" "); - } else { - sb.append("[").append(prefixX).append("]"); - } - } - } - if (sb.length() > 0) { - prefix = sb.toString(); - } - } - return prefix; - } - - /** - * Set the level of the logger. If the new level is null, the logger will inherit it's level from its nearest ancestor with a non-null - * level. - */ - public static void setLevel(Logger logger, String level) { - final Level l; - if (level == null) { - l = null; - } else { - l = Level.valueOf(level); - } - setLevel(logger, l); - } - - public static void setLevel(Logger logger, Level level) { -// if (!LogManager.ROOT_LOGGER_NAME.equals(logger.getName())) { -// -// Configurator.setLevel(logger.getName(), level); -// } else { -// -// final LoggerContext ctx = LoggerContext.getContext(false); -// final Configuration config = ctx.getConfiguration(); -// final LoggerConfig loggerConfig = config.getLoggerConfig(logger.getName()); -// loggerConfig.setLevel(level); -// ctx.updateLoggers(); -// } -// -// // we have to descend the hierarchy -// final LoggerContext ctx = LoggerContext.getContext(false); -// for (final LoggerConfig loggerConfig : ctx.getConfiguration().getLoggers().values()) { -// if (LogManager.ROOT_LOGGER_NAME.equals(logger.getName()) || loggerConfig.getName().startsWith(logger.getName() + ".")) { -// Configurator.setLevel(loggerConfig.getName(), level); -// } -// } - } - - public static void addAppender(final Logger logger, final Appender appender) { -// final LoggerContext ctx = (LoggerContext) LogManager.getContext(false); -// final Configuration config = ctx.getConfiguration(); -// config.addAppender(appender); -// LoggerConfig loggerConfig = config.getLoggerConfig(logger.getName()); -// if (!logger.getName().equals(loggerConfig.getName())) { -// loggerConfig = new LoggerConfig(logger.getName(), logger.getLevel(), true); -// config.addLogger(logger.getName(), loggerConfig); -// } -// loggerConfig.addAppender(appender, null, null); -// ctx.updateLoggers(); - } - - public static void removeAppender(final Logger logger, final Appender appender) { -// final LoggerContext ctx = (LoggerContext) LogManager.getContext(false); -// final Configuration config = ctx.getConfiguration(); -// LoggerConfig loggerConfig = config.getLoggerConfig(logger.getName()); -// if (!logger.getName().equals(loggerConfig.getName())) { -// loggerConfig = new LoggerConfig(logger.getName(), logger.getLevel(), true); -// config.addLogger(logger.getName(), loggerConfig); -// } -// loggerConfig.removeAppender(appender.getName()); -// ctx.updateLoggers(); - } - - public static Appender findAppender(final Logger logger, final Class clazz) { -// final LoggerContext ctx = (LoggerContext) LogManager.getContext(false); -// final Configuration config = ctx.getConfiguration(); -// final LoggerConfig loggerConfig = config.getLoggerConfig(logger.getName()); -// for (final Map.Entry entry : loggerConfig.getAppenders().entrySet()) { -// if (entry.getValue().getClass().equals(clazz)) { -// return entry.getValue(); -// } -// } - return null; - } - -} \ No newline at end of file diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/bucket/AggregationResponseHandle.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/bucket/AggregationResponseHandle.java deleted file mode 100644 index a284f906..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/bucket/AggregationResponseHandle.java +++ /dev/null @@ -1,171 +0,0 @@ -package org.jetlinks.community.elastic.search.aggreation.bucket; - -import org.elasticsearch.search.aggregations.Aggregation; -import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; -import org.elasticsearch.search.aggregations.bucket.range.Range; -import org.elasticsearch.search.aggregations.bucket.terms.Terms; -import org.elasticsearch.search.aggregations.metrics.*; -import org.jetlinks.community.elastic.search.aggreation.metrics.MetricsResponseSingleValue; - -import java.util.List; -import java.util.stream.Collectors; - -/** - * @author bsetfeng - * @since 1.0 - **/ -public class AggregationResponseHandle { - - - public static List terms(A a) { - Terms terms = (Terms) a; - return terms.getBuckets() - .stream() - .map(b -> { - Bucket bucket = Bucket.builder() - .key(b.getKeyAsString()) - .count(b.getDocCount()) - .name(a.getName()) - .build(); - b.getAggregations().asList() - .forEach(subAggregation -> route(bucket, subAggregation)); - return bucket; - }).collect(Collectors.toList()) - ; - } - - public static List range(A a) { - Range range = (Range) a; - return range.getBuckets() - .stream() - .map(b -> { - Bucket bucket = Bucket.builder() - .key(b.getKeyAsString()) - .from(b.getFrom()) - .to(b.getTo()) - .fromAsString(b.getFromAsString()) - .toAsString(b.getToAsString()) - .count(b.getDocCount()).build(); - b.getAggregations().asList() - .forEach(subAggregation -> { - route(bucket, subAggregation); - }); - return bucket; - }).collect(Collectors.toList()) - ; - } - - public static List dateHistogram(A a) { - Histogram histogram = (Histogram) a; - return bucketsHandle(histogram.getBuckets(), a.getName()); - } - - private static List bucketsHandle(List buckets, String name) { - return buckets - .stream() - .map(b -> { - Bucket bucket = Bucket.builder() - .key(b.getKeyAsString()) - .count(b.getDocCount()) - .name(name) - .build(); - b.getAggregations().asList() - .forEach(subAggregation -> route(bucket, subAggregation)); - return bucket; - }).collect(Collectors.toList()) - ; - } - - private static void route(Bucket bucket, A a) { - if (a instanceof Terms) { - bucket.setBuckets(terms(a)); - } else if (a instanceof Range) { - bucket.setBuckets(range(a)); - } else if (a instanceof Histogram) { - bucket.setBuckets(range(a)); - } else if (a instanceof Avg) { - bucket.setAvg(avg(a)); - } else if (a instanceof Min) { - bucket.setMin(min(a)); - } else if (a instanceof Max) { - bucket.setMax(max(a)); - } else if (a instanceof Sum) { - bucket.setSum(sum(a)); - } else if (a instanceof Stats) { - stats(bucket, a); - } else if (a instanceof ValueCount) { - bucket.setValueCount(count(a)); - } else { - throw new UnsupportedOperationException("不支持的聚合类型"); - } - } - - public static MetricsResponseSingleValue count(A a) { - ValueCount max = (ValueCount) a; - return MetricsResponseSingleValue.builder() - .value(max.getValue()) - .name(a.getName()) - .valueAsString(max.getValueAsString()) - .build(); - } - - public static MetricsResponseSingleValue avg(A a) { - Avg avg = (Avg) a; - return MetricsResponseSingleValue.builder() - .value(avg.getValue()) - .name(a.getName()) - .valueAsString(avg.getValueAsString()) - .build(); - } - - public static MetricsResponseSingleValue max(A a) { - Max max = (Max) a; - return MetricsResponseSingleValue.builder() - .value(max.getValue()) - .name(a.getName()) - .valueAsString(max.getValueAsString()) - .build(); - } - - public static MetricsResponseSingleValue min(A a) { - Min min = (Min) a; - return MetricsResponseSingleValue.builder() - .value(min.getValue()) - .name(a.getName()) - .valueAsString(min.getValueAsString()) - .build(); - } - - public static MetricsResponseSingleValue sum(A a) { - Sum sum = (Sum) a; - return MetricsResponseSingleValue.builder() - .value(sum.getValue()) - .name(a.getName()) - .valueAsString(sum.getValueAsString()) - .build(); - } - - public static void stats(Bucket bucket, A a) { - Stats stats = (Stats) a; - bucket.setAvg(MetricsResponseSingleValue.builder() - .value(stats.getAvg()) - .name(a.getName()) - .valueAsString(stats.getAvgAsString()) - .build()); - bucket.setMax(MetricsResponseSingleValue.builder() - .value(stats.getMax()) - .name(a.getName()) - .valueAsString(stats.getMaxAsString()) - .build()); - bucket.setMin(MetricsResponseSingleValue.builder() - .value(stats.getMin()) - .name(a.getName()) - .valueAsString(stats.getMinAsString()) - .build()); - bucket.setSum(MetricsResponseSingleValue.builder() - .value(stats.getSum()) - .name(a.getName()) - .valueAsString(stats.getSumAsString()) - .build()); - } -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/bucket/Bucket.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/bucket/Bucket.java deleted file mode 100644 index a79e1b12..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/bucket/Bucket.java +++ /dev/null @@ -1,70 +0,0 @@ -package org.jetlinks.community.elastic.search.aggreation.bucket; - -import lombok.*; -import org.jetlinks.community.elastic.search.aggreation.metrics.MetricsResponseSingleValue; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * @author bsetfeng - * @since 1.0 - **/ -@Getter -@Setter -@AllArgsConstructor -@NoArgsConstructor -@Builder -public class Bucket { - - private String key; - - private String name; - - private long count; - - private String fromAsString; - - private Object from; - - private String toAsString; - - private Object to; - - private MetricsResponseSingleValue sum; - - private MetricsResponseSingleValue valueCount; - - private MetricsResponseSingleValue avg; - - private MetricsResponseSingleValue min; - - private MetricsResponseSingleValue max; - - private List buckets; - - private double toNumber(double number) { - return (Double.isInfinite(number) || Double.isNaN(number)) ? 0 : number; - } - - public Map toMap() { - Map map = new HashMap<>(); - if (this.sum != null) { - map.put(sum.getName(), toNumber(sum.getValue())); - } - if (this.valueCount != null) { - map.put(valueCount.getName(), toNumber(valueCount.getValue())); - } - if (this.avg != null) { - map.put(avg.getName(), toNumber(avg.getValue())); - } - if (this.min != null) { - map.put(min.getName(), toNumber(min.getValue())); - } - if (this.max != null) { - map.put(max.getName(), toNumber(max.getValue())); - } - return map; - } -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/bucket/BucketAggregationsStructure.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/bucket/BucketAggregationsStructure.java deleted file mode 100644 index 7ddef780..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/bucket/BucketAggregationsStructure.java +++ /dev/null @@ -1,70 +0,0 @@ -package org.jetlinks.community.elastic.search.aggreation.bucket; - -import lombok.*; -import org.elasticsearch.search.aggregations.bucket.histogram.LongBounds; -import org.hswebframework.utils.StringUtils; -import org.jetlinks.community.elastic.search.aggreation.enums.BucketType; -import org.jetlinks.community.elastic.search.aggreation.metrics.MetricsAggregationStructure; - -import java.util.LinkedList; -import java.util.List; - -/** - * @author bsetfeng - * @since 1.0 - **/ -@Setter -@Getter -@Builder -@AllArgsConstructor -@NoArgsConstructor -public class BucketAggregationsStructure { - - @NonNull - private String field; - - private String name; - - @NonNull - private BucketType type = BucketType.TERMS; - - /** - * 指定返回分组数量 - */ - private Integer size; - - private Sort sort; - - private List ranges; - - private LongBounds extendedBounds; - - /** - * 时间格式 - */ - private String format; - - - /** - * 单位时间间隔 - * - * @see DateHistogramInterval - */ - private String interval; - - /** - * 缺失值 - */ - private Object missingValue; - - private List subMetricsAggregation = new LinkedList<>(); - - private List subBucketAggregation = new LinkedList<>(); - - public String getName() { - if (StringUtils.isNullOrEmpty(name)) { - name = type.name().concat("_").concat(field); - } - return name; - } -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/bucket/BucketResponse.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/bucket/BucketResponse.java deleted file mode 100644 index e69452b4..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/bucket/BucketResponse.java +++ /dev/null @@ -1,21 +0,0 @@ -package org.jetlinks.community.elastic.search.aggreation.bucket; - -import lombok.*; - -import java.util.List; - -/** - * @author bsetfeng - * @since 1.0 - **/ -@Getter -@Setter -@AllArgsConstructor -@NoArgsConstructor -@Builder -public class BucketResponse { - - private String name; - - private List buckets; -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/bucket/DateHistogramInterval.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/bucket/DateHistogramInterval.java deleted file mode 100644 index ce94b717..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/bucket/DateHistogramInterval.java +++ /dev/null @@ -1,38 +0,0 @@ - -package org.jetlinks.community.elastic.search.aggreation.bucket; - -/** - * The interval the date histogram is based on. - */ -public class DateHistogramInterval { - - public static final String SECOND = "1s"; - public static final String MINUTE = "1m"; - public static final String HOUR = "1h"; - public static final String DAY = "1d"; - public static final String WEEK = "1w"; - public static final String MONTH = "1M"; - public static final String QUARTER = "1q"; - public static final String YEAR = "1y"; - - public static String seconds(int sec) { - return sec + "s"; - } - - public static String minutes(int min) { - return min + "m"; - } - - public static String hours(int hours) { - return hours + "h"; - } - - public static String days(int days) { - return days + "d"; - } - - public static String weeks(int weeks) { - return weeks + "w"; - } - -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/bucket/Ranges.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/bucket/Ranges.java deleted file mode 100644 index 819a6e8b..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/bucket/Ranges.java +++ /dev/null @@ -1,19 +0,0 @@ -package org.jetlinks.community.elastic.search.aggreation.bucket; - -import lombok.Getter; -import lombok.Setter; - -/** - * @author bsetfeng - * @since 1.0 - **/ -@Getter -@Setter -public class Ranges { - - private String key; - - private Object form; - - private Object to; -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/bucket/Sort.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/bucket/Sort.java deleted file mode 100644 index 5b19526d..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/bucket/Sort.java +++ /dev/null @@ -1,54 +0,0 @@ -package org.jetlinks.community.elastic.search.aggreation.bucket; - -import lombok.Getter; -import lombok.Setter; -import org.jetlinks.community.elastic.search.aggreation.enums.OrderType; - -/** - * @author bsetfeng - * @since 1.0 - **/ -@Getter -@Setter -public class Sort { - - private String order; - - private OrderType type = OrderType.COUNT; - - private Sort() { - } - - private Sort(String order, OrderType type) { - this.type = type; - this.order = order; - } - - private Sort(String order) { - this.order = order; - } - - public String getOrder() { - if ("desc".equalsIgnoreCase(order)) { - return order; - } else { - return order = "asc"; - } - } - - public static Sort asc() { - return new Sort("asc"); - } - - public static Sort asc(OrderType type) { - return new Sort("asc", type); - } - - public static Sort desc() { - return new Sort("desc"); - } - - public static Sort desc(OrderType type) { - return new Sort("desc", type); - } -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/enums/AggregationType.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/enums/AggregationType.java deleted file mode 100644 index 09a2bf69..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/enums/AggregationType.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.jetlinks.community.elastic.search.aggreation.enums; - -import lombok.AllArgsConstructor; -import lombok.Getter; - -/** - * @author bsetfeng - * @since 1.0 - **/ -@Getter -@AllArgsConstructor -public enum AggregationType { - AVG("平均"), - MAX("最大"), - COUNT("计数"), - MIN("最小"), - SUM("总数"), - STATS("统计汇总"), - EXTENDED_STATS("扩展统计"), - CARDINALITY("基数"),//去重统计 - VALUE_COUNT("非空值计数"), - TERMS("字段项"), - RANGE("范围"), - DATE_HISTOGRAM("直方图"), - DATE_RANGE("时间范围"); - - private String text; -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/enums/BucketType.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/enums/BucketType.java deleted file mode 100644 index b8337454..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/enums/BucketType.java +++ /dev/null @@ -1,199 +0,0 @@ -package org.jetlinks.community.elastic.search.aggreation.enums; - -import lombok.AllArgsConstructor; -import lombok.EqualsAndHashCode; -import lombok.Getter; -import org.elasticsearch.search.aggregations.Aggregation; -import org.elasticsearch.search.aggregations.AggregationBuilder; -import org.elasticsearch.search.aggregations.AggregationBuilders; -import org.elasticsearch.search.aggregations.BucketOrder; -import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; -import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; -import org.elasticsearch.search.aggregations.bucket.range.DateRangeAggregationBuilder; -import org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder; -import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; -import org.jetlinks.community.elastic.search.aggreation.bucket.AggregationResponseHandle; -import org.jetlinks.community.elastic.search.aggreation.bucket.Bucket; -import org.jetlinks.community.elastic.search.aggreation.bucket.BucketAggregationsStructure; -import org.jetlinks.community.elastic.search.aggreation.bucket.Sort; -import org.jetlinks.community.elastic.search.aggreation.metrics.MetricsAggregationStructure; -import org.springframework.util.StringUtils; - -import java.time.ZoneId; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * @author bsetfeng - * @since 1.0 - **/ -@Getter -@AllArgsConstructor -public enum BucketType { - - - TERMS("字段项") { - @Override - public AggregationBuilder aggregationBuilder(BucketAggregationsStructure structure) { - TermsAggregationBuilder builder = AggregationBuilders - .terms(structure.getName()) - .field(structure.getField()); - if (structure.getSize() != null) { - builder.size(structure.getSize()); - } - Sort sort = structure.getSort(); - if (sort != null) { - builder.order(mapping.get(OrderBuilder.of(sort.getOrder(), sort.getType()))); - } - if (structure.getMissingValue() != null) { - builder.missing(structure.getMissingValue()); - } - commonAggregationSetting(builder, structure); - return builder; - } - - @Override - public List convert(A a) { - return AggregationResponseHandle.terms(a); - } - }, - RANGE("范围") { - @Override - public AggregationBuilder aggregationBuilder(BucketAggregationsStructure structure) { - RangeAggregationBuilder builder = AggregationBuilders - .range(structure.getName()) - .field(structure.getField()); - if (StringUtils.hasText(structure.getFormat())) { - String format = structure.getFormat(); - if (format.startsWith("yyyy")) { - format = "8" + format; - } - builder.format(format); - } - structure.getRanges() - .forEach(ranges -> { - builder.addRange(ranges.getKey(), (Double) ranges.getForm(), (Double) ranges.getTo()); - }); - commonAggregationSetting(builder, structure); - return builder; - } - - @Override - public List convert(A a) { - return AggregationResponseHandle.range(a); - } - }, - DATE_RANGE("时间范围") { - @Override - public AggregationBuilder aggregationBuilder(BucketAggregationsStructure structure) { - DateRangeAggregationBuilder builder = AggregationBuilders - .dateRange(structure.getName()) - .field(structure.getField()); - if (StringUtils.hasText(structure.getFormat())) { - String format = structure.getFormat(); - if (format.startsWith("yyyy")) { - format = "8" + format; - } - builder.format(format); - } - structure.getRanges() - .forEach(ranges -> { - builder.addRange(ranges.getKey(), ranges.getForm().toString(), ranges.getTo().toString()); - }); - if (structure.getMissingValue() != null) { - builder.missing(structure.getMissingValue()); - } - builder.timeZone(ZoneId.systemDefault()); - commonAggregationSetting(builder, structure); - return builder; - } - - @Override - public List convert(A a) { - return AggregationResponseHandle.range(a); - } - }, - DATE_HISTOGRAM("时间范围") { - @Override - public AggregationBuilder aggregationBuilder(BucketAggregationsStructure structure) { - DateHistogramAggregationBuilder builder = AggregationBuilders - .dateHistogram(structure.getName()) - .field(structure.getField()); - if (StringUtils.hasText(structure.getFormat())) { - builder.format(structure.getFormat()); - } - if (StringUtils.hasText(structure.getInterval())) { - builder.dateHistogramInterval(new DateHistogramInterval(structure.getInterval())); - } - if (structure.getExtendedBounds() != null) { - builder.extendedBounds(structure.getExtendedBounds()); - } - if (structure.getMissingValue() != null) { - builder.missing(structure.getMissingValue()); - } - Sort sort = structure.getSort(); - if (sort != null) { - builder.order(mapping.get(OrderBuilder.of(sort.getOrder(), sort.getType()))); - } - builder.timeZone(ZoneId.systemDefault()); - commonAggregationSetting(builder, structure); - return builder; - } - - - @Override - public List convert(A a) { - return AggregationResponseHandle.dateHistogram(a); - } - }; - - private final String text; - - public abstract AggregationBuilder aggregationBuilder(BucketAggregationsStructure structure); - - public abstract List convert(A a); - - private static void commonAggregationSetting(AggregationBuilder builder, BucketAggregationsStructure structure) { - if (structure.getSubMetricsAggregation() != null && structure.getSubMetricsAggregation().size() > 0) { - addMetricsSubAggregation(builder, structure.getSubMetricsAggregation()); - } - if (structure.getSubBucketAggregation() != null && structure.getSubBucketAggregation().size() > 0) { - addBucketSubAggregation(builder, structure.getSubBucketAggregation()); - } - } - - private static void addMetricsSubAggregation(AggregationBuilder builder, List subMetricsAggregation) { - subMetricsAggregation - .forEach(subStructure -> { - builder.subAggregation(subStructure.getType().aggregationBuilder(subStructure.getName(), subStructure.getField())); - }); - } - - private static void addBucketSubAggregation(AggregationBuilder builder, List subBucketAggregation) { - subBucketAggregation - .forEach(subStructure -> { - builder.subAggregation(subStructure.getType().aggregationBuilder(subStructure)); - }); - } - - @Getter - @AllArgsConstructor(staticName = "of") - @EqualsAndHashCode - static class OrderBuilder { - - private String order; - - private OrderType orderType; - } - - static Map mapping = new HashMap<>(); - - static { - mapping.put(OrderBuilder.of("asc", OrderType.COUNT), BucketOrder.count(true)); - mapping.put(OrderBuilder.of("desc", OrderType.COUNT), BucketOrder.count(false)); - mapping.put(OrderBuilder.of("asc", OrderType.KEY), BucketOrder.key(true)); - mapping.put(OrderBuilder.of("desc", OrderType.KEY), BucketOrder.key(false)); - } - -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/enums/MetricsType.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/enums/MetricsType.java deleted file mode 100644 index 090e12a3..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/enums/MetricsType.java +++ /dev/null @@ -1,136 +0,0 @@ -package org.jetlinks.community.elastic.search.aggreation.enums; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.search.aggregations.AggregationBuilder; -import org.elasticsearch.search.aggregations.AggregationBuilders; -import org.elasticsearch.search.aggregations.metrics.*; -import org.jetlinks.community.elastic.search.aggreation.metrics.MetricsResponse; -import org.jetlinks.community.elastic.search.aggreation.metrics.MetricsResponseSingleValue; - -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -/** - * @author bsetfeng - * @since 1.0 - **/ -@Getter -@AllArgsConstructor -public enum MetricsType { - - AVG("平均") { - @Override - public AggregationBuilder aggregationBuilder(String name, String filed) { - return AggregationBuilders.avg(name).field(filed); - } - - @Override - public MetricsResponse getResponse(String name, SearchResponse response) { - Avg avg = response.getAggregations().get(name); - return MetricsResponse.builder() - .results(Collections.singletonMap(AVG, - new MetricsResponseSingleValue(avg.getValue(), avg.getName(), avg.getValueAsString()))) - .build(); - } - }, - MAX("最大") { - @Override - public AggregationBuilder aggregationBuilder(String name, String filed) { - return AggregationBuilders.max(name).field(filed); - } - - @Override - public MetricsResponse getResponse(String name, SearchResponse response) { - Max max = response.getAggregations().get(name); - return MetricsResponse.builder() - .results(Collections.singletonMap(MAX, - new MetricsResponseSingleValue(max.getValue(), max.getName(), max.getValueAsString()))) - .build(); - } - }, - COUNT("非空值计数") { - @Override - public AggregationBuilder aggregationBuilder(String name, String filed) { - return AggregationBuilders.count(name).field(filed); - } - - @Override - public MetricsResponse getResponse(String name, SearchResponse response) { - ValueCount valueCount = response.getAggregations().get(name); - return MetricsResponse.builder() - .results(Collections.singletonMap(COUNT, - new MetricsResponseSingleValue(valueCount.getValue(), valueCount.getName(), valueCount.getValueAsString()))) - .build(); - } - }, - MIN("最小") { - @Override - public AggregationBuilder aggregationBuilder(String name, String filed) { - return AggregationBuilders.min(name).field(filed); - } - - @Override - public MetricsResponse getResponse(String name, SearchResponse response) { - Min min = response.getAggregations().get(name); - return MetricsResponse.builder() - .results(Collections.singletonMap(MIN, - new MetricsResponseSingleValue(min.getValue(), min.getName(), min.getValueAsString()))) - .build(); - } - }, - SUM("总数") { - @Override - public AggregationBuilder aggregationBuilder(String name, String filed) { - return AggregationBuilders.sum(name).field(filed); - } - - @Override - public MetricsResponse getResponse(String name, SearchResponse response) { - Sum sum = response.getAggregations().get(name); - return MetricsResponse.builder() - .results(Collections.singletonMap(SUM, - new MetricsResponseSingleValue(sum.getValue(), sum.getName(), sum.getValueAsString()))) - .build(); - } - }, - STATS("统计汇总") { - @Override - public AggregationBuilder aggregationBuilder(String name, String filed) { - return AggregationBuilders.stats(name).field(filed); - } - - @Override - public MetricsResponse getResponse(String name, SearchResponse response) { - Stats stats = response.getAggregations().get(name); - Map results = new HashMap<>(); - results.put(AVG, new MetricsResponseSingleValue(stats.getAvg(), stats.getName(), stats.getAvgAsString())); - results.put(MIN, new MetricsResponseSingleValue(stats.getMin(), stats.getName(), stats.getMinAsString())); - results.put(MAX, new MetricsResponseSingleValue(stats.getMax(), stats.getName(), stats.getMaxAsString())); - results.put(SUM, new MetricsResponseSingleValue(stats.getSum(), stats.getName(), stats.getMaxAsString())); - results.put(COUNT, new MetricsResponseSingleValue(stats.getCount(), stats.getName(), String.valueOf(stats.getCount()))); - return MetricsResponse.builder() - .results(results) - .build(); - } - }; - - private String text; - - - public abstract AggregationBuilder aggregationBuilder(String name, String filed); - - public abstract MetricsResponse getResponse(String name, SearchResponse response); - - public static MetricsType of(String name) { - for (MetricsType type : MetricsType.values()) { - if (type.name().equalsIgnoreCase(name)) { - return type; - } - } - throw new UnsupportedOperationException("不支持的聚合度量类型：" + name); - } - -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/enums/OrderType.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/enums/OrderType.java deleted file mode 100644 index e580b939..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/enums/OrderType.java +++ /dev/null @@ -1,18 +0,0 @@ -package org.jetlinks.community.elastic.search.aggreation.enums; - -import lombok.AllArgsConstructor; -import lombok.Getter; - -/** - * @author bsetfeng - * @since 1.0 - **/ -@AllArgsConstructor -@Getter -public enum OrderType { - - COUNT("计数"), - KEY("分组值"); - - private String text; -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/metrics/MetricsAggregationStructure.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/metrics/MetricsAggregationStructure.java deleted file mode 100644 index 6d689c02..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/metrics/MetricsAggregationStructure.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.jetlinks.community.elastic.search.aggreation.metrics; - -import lombok.*; -import org.hswebframework.utils.StringUtils; -import org.jetlinks.community.elastic.search.aggreation.enums.MetricsType; - -/** - * @author bsetfeng - * @since 1.0 - **/ -@Setter -@Getter -@Builder -@AllArgsConstructor -@NoArgsConstructor -public class MetricsAggregationStructure { - - @NonNull - private String field; - - private String name; - - @NonNull - private MetricsType type = MetricsType.COUNT; - - /** - * 缺失值 - */ - private Object missingValue; - - public String getName() { - if (StringUtils.isNullOrEmpty(name)) { - name = type.name().concat("_").concat(field); - } - return name; - } -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/metrics/MetricsResponse.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/metrics/MetricsResponse.java deleted file mode 100644 index 8aafbb40..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/metrics/MetricsResponse.java +++ /dev/null @@ -1,33 +0,0 @@ -package org.jetlinks.community.elastic.search.aggreation.metrics; - -import lombok.*; -import org.jetlinks.community.elastic.search.aggreation.enums.MetricsType; - -import java.util.Map; - -/** - * @author bsetfeng - * @since 1.0 - **/ -@Getter -@Setter -@AllArgsConstructor -@NoArgsConstructor -@Builder -public class MetricsResponse { - - private Map results; - - private MetricsResponseSingleValue singleResult; - - public MetricsResponseSingleValue getSingleResult() { - if (singleResult == null) { - this.singleResult = results.entrySet() - .stream() - .findFirst() - .map(Map.Entry::getValue) - .orElse(MetricsResponseSingleValue.empty()); - } - return singleResult; - } -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/metrics/MetricsResponseSingleValue.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/metrics/MetricsResponseSingleValue.java deleted file mode 100644 index c5855460..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/aggreation/metrics/MetricsResponseSingleValue.java +++ /dev/null @@ -1,47 +0,0 @@ -package org.jetlinks.community.elastic.search.aggreation.metrics; - -import lombok.*; - -/** - * @author bsetfeng - * @since 1.0 - **/ - -@Setter -@Getter -@AllArgsConstructor -@NoArgsConstructor -@Builder -public class MetricsResponseSingleValue { - - private double value; - - private String valueAsString; - - private String name; - - public static MetricsResponseSingleValue empty() { - return new MetricsResponseSingleValue(); - } - -// public double getValue() { -// if (isDoubleOrFloat(String.valueOf(value))) { -// BigDecimal b = BigDecimal.valueOf(value); -// return b.setScale(2, BigDecimal.ROUND_HALF_UP).doubleValue(); -// } else { -// return 0; -// } -// } -// -// public static boolean isDoubleOrFloat(String str) { -// if (str.length() > 15) { -// str = str.substring(0, 15); -// } -// Pattern pattern = Pattern.compile("^[-\\+]?[.\\d]*$"); -// return pattern.matcher(str).matches(); -// } -// -// public static void main(String[] args) { -// System.out.println(new BigDecimal("8.839927333333333E7")); -// } -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/configuration/ElasticSearchProperties.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/configuration/ElasticSearchProperties.java deleted file mode 100644 index 1ddefa42..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/configuration/ElasticSearchProperties.java +++ /dev/null @@ -1,51 +0,0 @@ -package org.jetlinks.community.elastic.search.configuration; - -import lombok.Getter; -import lombok.Setter; -import org.apache.commons.collections.CollectionUtils; -import org.apache.http.HttpHost; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; -import org.springframework.boot.context.properties.ConfigurationProperties; - -import java.util.List; - -@ConfigurationProperties(prefix = "elasticsearch.client") - -@Getter -@Setter -public class ElasticSearchProperties { - - private String host = "localhost"; - private int port = 9200; - - private int connectionRequestTimeout = 5000; - private int connectTimeout = 2000; - private int socketTimeout = 2000; - private int maxConnTotal = 30; - private List hosts; - - - public HttpHost[] createHosts() { - if (CollectionUtils.isEmpty(hosts)) { - return new HttpHost[]{new HttpHost(host, port, "http")}; - } - - return hosts.stream().map(HttpHost::create).toArray(HttpHost[]::new); - } - - public RequestConfig.Builder applyRequestConfigBuilder(RequestConfig.Builder builder) { - - builder.setConnectTimeout(connectTimeout); - builder.setConnectionRequestTimeout(connectionRequestTimeout); - builder.setSocketTimeout(socketTimeout); - - return builder; - } - - public HttpAsyncClientBuilder applyHttpAsyncClientBuilder(HttpAsyncClientBuilder builder) { - builder.setMaxConnTotal(maxConnTotal); - - return builder; - } -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/embedded/EmbeddedElasticSearch.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/embedded/EmbeddedElasticSearch.java deleted file mode 100644 index fbd01801..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/embedded/EmbeddedElasticSearch.java +++ /dev/null @@ -1,33 +0,0 @@ -package org.jetlinks.community.elastic.search.embedded; - -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.node.InternalSettingsPreparer; -import org.elasticsearch.node.Node; -import org.elasticsearch.transport.Netty4Plugin; - -import java.util.Collections; - -@Slf4j -public class EmbeddedElasticSearch extends Node { - - static { - System.setProperty("es.set.netty.runtime.available.processors","false"); - } - @SneakyThrows - public EmbeddedElasticSearch(EmbeddedElasticSearchProperties properties) { - super(InternalSettingsPreparer.prepareEnvironment( - properties.applySetting( - Settings.builder() - .put("node.name", "test") - .put("discovery.type", "single-node") - .put("transport.type", Netty4Plugin.NETTY_TRANSPORT_NAME) - .put("http.type", Netty4Plugin.NETTY_HTTP_TRANSPORT_NAME) - .put("network.host", "0.0.0.0") - .put("http.port", 9200) - ).build(), Collections.emptyMap(), null, () -> "default"), - Collections.singleton(Netty4Plugin.class), false); - } - -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/embedded/EmbeddedElasticSearchProperties.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/embedded/EmbeddedElasticSearchProperties.java deleted file mode 100644 index 8931e2e8..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/embedded/EmbeddedElasticSearchProperties.java +++ /dev/null @@ -1,30 +0,0 @@ -package org.jetlinks.community.elastic.search.embedded; - -import lombok.Getter; -import lombok.Setter; -import org.elasticsearch.common.settings.Settings; -import org.springframework.boot.context.properties.ConfigurationProperties; - -@ConfigurationProperties(prefix = "elasticsearch.embedded") -@Getter -@Setter -public class EmbeddedElasticSearchProperties { - - private boolean enabled; - - private String dataPath = "./data/elasticsearch"; - - private String homePath = "./"; - - private int port = 9200; - - private String host = "0.0.0.0"; - - - public Settings.Builder applySetting(Settings.Builder settings) { - return settings.put("network.host", host) - .put("http.port", port) - .put("path.data", dataPath) - .put("path.home", homePath); - } -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/enums/LinkTypeEnum.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/enums/LinkTypeEnum.java deleted file mode 100644 index 5fb8dc0b..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/enums/LinkTypeEnum.java +++ /dev/null @@ -1,69 +0,0 @@ -package org.jetlinks.community.elastic.search.enums; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.hswebframework.ezorm.core.param.Term; - -import java.util.Arrays; -import java.util.LinkedList; -import java.util.Optional; - -/** - * @author Jia_RG - */ -@Getter -@AllArgsConstructor -public enum LinkTypeEnum { - and("and") { - @Override - public void process(BoolQueryBuilder query, Term term) { - if (term.getTerms().isEmpty()) { - query.must(TermTypeEnum.of(term.getTermType().trim()).map(e -> e.process(term)).orElse(QueryBuilders.boolQuery())); - } else { - // 嵌套查询新建一个包起来 - BoolQueryBuilder nextQuery = QueryBuilders.boolQuery(); - LinkedList terms = ((LinkedList) term.getTerms()); - // 同一层级取最后一个的type - LinkTypeEnum.of(getLast(terms).getType().name()).ifPresent(e -> terms.forEach(t -> e.process(nextQuery, t))); - // 处理完后包括进去 - query.must(nextQuery); - } - } - }, - or("or") { - @Override - public void process(BoolQueryBuilder query, Term term) { - // 跟上面代码相似 - if (term.getTerms().isEmpty()) { - query.should(TermTypeEnum.of(term.getTermType().trim()).map(e -> e.process(term)).orElse(QueryBuilders.boolQuery())); - } else { - BoolQueryBuilder nextQuery = QueryBuilders.boolQuery(); - LinkedList terms = ((LinkedList) term.getTerms()); - LinkTypeEnum.of(getLast(terms).getType().name()).ifPresent(e -> terms.forEach(t -> e.process(nextQuery, t))); - query.should(nextQuery); - } - } - }; - - private final String type; - - public abstract void process(BoolQueryBuilder query, Term term); - - - public static Optional of(String type) { - return Arrays.stream(values()) - .filter(e -> e.getType().equalsIgnoreCase(type)) - .findAny(); - } - - private static Term getLast(LinkedList terms) { - int index = terms.indexOf(terms.getLast()); - while (index >= 0) { - if (terms.get(index).getTerms().isEmpty()) break; - index--; - } - return terms.get(index); - } -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/enums/TermTypeEnum.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/enums/TermTypeEnum.java deleted file mode 100644 index 7c43f8bc..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/enums/TermTypeEnum.java +++ /dev/null @@ -1,114 +0,0 @@ -package org.jetlinks.community.elastic.search.enums; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.hswebframework.ezorm.core.param.Term; -import org.jetlinks.community.elastic.search.utils.TermCommonUtils; -import org.jetlinks.reactor.ql.utils.CastUtils; -import org.springframework.util.StringUtils; - -import java.util.Arrays; -import java.util.List; -import java.util.Optional; - -/** - * @author Jia_RG - * @author bestfeng - */ -@Getter -@AllArgsConstructor -public enum TermTypeEnum { - eq("eq") { - @Override - public QueryBuilder process(Term term) { - return QueryBuilders.termQuery(term.getColumn().trim(), term.getValue()); - } - }, - not("not") { - @Override - public QueryBuilder process(Term term) { - return QueryBuilders.boolQuery().mustNot(QueryBuilders.termQuery(term.getColumn().trim(), term.getValue())); - } - }, - btw("btw") { - @Override - public QueryBuilder process(Term term) { - Object between = null; - Object and = null; - List values = TermCommonUtils.convertToList(term.getValue()); - if (values.size() > 0) { - between = CastUtils.castNumber(values.get(0)); - } - if (values.size() > 1) { - and = CastUtils.castNumber(values.get(1)); - } - return QueryBuilders.rangeQuery(term.getColumn().trim()).gte(between).lte(and); - } - }, - gt("gt") { - @Override - public QueryBuilder process(Term term) { - Object value = CastUtils.castNumber(term.getValue()); - return QueryBuilders.rangeQuery(term.getColumn().trim()).gt(value); - } - }, - gte("gte") { - @Override - public QueryBuilder process(Term term) { - Object value = CastUtils.castNumber(term.getValue()); - return QueryBuilders.rangeQuery(term.getColumn().trim()).gte(value); - } - }, - lt("lt") { - @Override - public QueryBuilder process(Term term) { - Object value = CastUtils.castNumber(term.getValue()); - return QueryBuilders.rangeQuery(term.getColumn().trim()).lt(value); - } - }, - lte("lte") { - @Override - public QueryBuilder process(Term term) { - Object value = CastUtils.castNumber(term.getValue()); - return QueryBuilders.rangeQuery(term.getColumn().trim()).lte(value); - } - }, - in("in") { - @Override - public QueryBuilder process(Term term) { - return QueryBuilders.termsQuery(term.getColumn().trim(), TermCommonUtils.convertToList(term.getValue())); - } - }, - like("like") { - @Override - public QueryBuilder process(Term term) { - //return QueryBuilders.matchPhraseQuery(term.getColumn().trim(), term.getValue()); - return QueryBuilders.wildcardQuery(term.getColumn().trim(), likeQueryTermValueHandler(term.getValue())); - } - }, - nlike("nlike") { - @Override - public QueryBuilder process(Term term) { - return QueryBuilders.boolQuery().mustNot(QueryBuilders.wildcardQuery(term.getColumn().trim(), likeQueryTermValueHandler(term.getValue()))); - } - }; - - private final String type; - - public abstract QueryBuilder process(Term term); - - public static String likeQueryTermValueHandler(Object value) { - if (!StringUtils.isEmpty(value)) { - return value.toString().replace("%", "*"); - } - return "**"; - } - - public static Optional of(String type) { - return Arrays.stream(values()) - .filter(e -> e.getType().equalsIgnoreCase(type)) - .findAny(); - } -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexProperties.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexProperties.java deleted file mode 100644 index b5fdabdc..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/ElasticSearchIndexProperties.java +++ /dev/null @@ -1,33 +0,0 @@ -package org.jetlinks.community.elastic.search.index; - -import lombok.*; -import org.elasticsearch.common.settings.Settings; -import org.springframework.boot.context.properties.ConfigurationProperties; - -import java.util.HashMap; -import java.util.Map; -import java.util.function.Function; - -@Getter -@Setter -@Builder -@NoArgsConstructor -@AllArgsConstructor -@ConfigurationProperties(prefix = "elasticsearch.index.settings") -public class ElasticSearchIndexProperties { - - private int numberOfShards = 1; - - private int numberOfReplicas = 0; - - private Map options = new HashMap<>(); - - public Settings toSettings() { - - return Settings.builder() - .put("number_of_shards", Math.max(1, numberOfShards)) - .put("number_of_replicas", numberOfReplicas) - .putProperties(options, Function.identity()) - .build(); - } -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/IndexTemplateProvider.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/IndexTemplateProvider.java deleted file mode 100644 index d7e8bca8..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/IndexTemplateProvider.java +++ /dev/null @@ -1,12 +0,0 @@ -package org.jetlinks.community.elastic.search.index; - -/** - * @author bsetfeng - * @since 1.0 - **/ -public interface IndexTemplateProvider { - - static String getIndexTemplate(String index) { - return index.concat("_template"); - } -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/mapping/IndexMappingMetadata.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/mapping/IndexMappingMetadata.java deleted file mode 100644 index 1490a52d..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/mapping/IndexMappingMetadata.java +++ /dev/null @@ -1,64 +0,0 @@ -package org.jetlinks.community.elastic.search.index.mapping; - -import lombok.Getter; -import lombok.Setter; -import org.jetlinks.community.elastic.search.enums.ElasticPropertyType; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.function.Function; -import java.util.stream.Collectors; - -/** - * @author bsetfeng - * @since 1.0 - **/ -@Getter -@Setter -public class IndexMappingMetadata { - - private String index; - - private Map metadata = new HashMap<>(); - - public List getAllMetaData() { - return metadata.entrySet() - .stream() - .map(Map.Entry::getValue) - .collect(Collectors.toList()); - } - - public SingleMappingMetadata getMetaData(String fieldName) { - return metadata.get(fieldName); - } - - public List getMetaDataByType(ElasticPropertyType type) { - return getAllMetaData() - .stream() - .filter(singleMapping -> singleMapping.getType().equals(type)) - .collect(Collectors.toList()); - } - - public Map getMetaDataByTypeToMap(ElasticPropertyType type) { - return getMetaDataByType(type) - .stream() - .collect(Collectors.toMap(SingleMappingMetadata::getName, Function.identity())); - } - - public void setMetadata(SingleMappingMetadata singleMapping) { - metadata.put(singleMapping.getName(), singleMapping); - } - - - private IndexMappingMetadata(String index) { - this.index = index; - } - - private IndexMappingMetadata() { - } - - public static IndexMappingMetadata getInstance(String index) { - return new IndexMappingMetadata(index); - } -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/mapping/SingleMappingMetadata.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/mapping/SingleMappingMetadata.java deleted file mode 100644 index 2349d26b..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/mapping/SingleMappingMetadata.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.jetlinks.community.elastic.search.index.mapping; - -import lombok.*; -import org.jetlinks.community.elastic.search.enums.ElasticDateFormat; -import org.jetlinks.community.elastic.search.enums.ElasticPropertyType; - -/** - * @author bsetfeng - * @since 1.0 - **/ -@Getter -@Setter -@Builder -@AllArgsConstructor -@NoArgsConstructor -public class SingleMappingMetadata { - - private String name; - - private ElasticDateFormat format; - - private ElasticPropertyType type; -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/strategies/AbstractElasticSearchIndexStrategy.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/strategies/AbstractElasticSearchIndexStrategy.java deleted file mode 100755 index deca56ea..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/index/strategies/AbstractElasticSearchIndexStrategy.java +++ /dev/null @@ -1,257 +0,0 @@ -package org.jetlinks.community.elastic.search.index.strategies; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.Version; -import org.elasticsearch.client.indices.CreateIndexRequest; -import org.elasticsearch.client.indices.GetIndexRequest; -import org.elasticsearch.client.indices.GetMappingsRequest; -import org.elasticsearch.client.indices.PutMappingRequest; -import org.elasticsearch.cluster.metadata.MappingMetadata; -import org.elasticsearch.common.compress.CompressedXContent; -import org.jetlinks.core.metadata.DataType; -import org.jetlinks.core.metadata.PropertyMetadata; -import org.jetlinks.core.metadata.SimplePropertyMetadata; -import org.jetlinks.core.metadata.types.*; -import org.jetlinks.community.elastic.search.enums.ElasticDateFormat; -import org.jetlinks.community.elastic.search.enums.ElasticPropertyType; -import org.jetlinks.community.elastic.search.index.DefaultElasticSearchIndexMetadata; -import org.jetlinks.community.elastic.search.index.ElasticSearchIndexMetadata; -import org.jetlinks.community.elastic.search.index.ElasticSearchIndexProperties; -import org.jetlinks.community.elastic.search.index.ElasticSearchIndexStrategy; -import org.jetlinks.community.elastic.search.service.reactive.ReactiveElasticsearchClient; -import org.springframework.util.CollectionUtils; -import reactor.core.publisher.Mono; - -import java.util.*; -import java.util.stream.Collectors; - -@AllArgsConstructor -@Slf4j -public abstract class AbstractElasticSearchIndexStrategy implements ElasticSearchIndexStrategy { - @Getter - private final String id; - - protected ReactiveElasticsearchClient client; - - protected ElasticSearchIndexProperties properties; - - protected String wrapIndex(String index) { - return index.toLowerCase(); - } - - protected Mono indexExists(String index) { - return client.existsIndex(new GetIndexRequest(wrapIndex(index))); - } - - protected Mono doCreateIndex(ElasticSearchIndexMetadata metadata) { - return client - .createIndex(createIndexRequest(metadata)) - .then(); - } - - protected Mono doPutIndex(ElasticSearchIndexMetadata metadata, - boolean justUpdateMapping) { - String index = wrapIndex(metadata.getIndex()); - return this.indexExists(index) - .flatMap(exists -> { - if (exists) { - return doLoadIndexMetadata(index) - .flatMap(oldMapping -> Mono.justOrEmpty(createPutMappingRequest(metadata, oldMapping))) - .flatMap(request -> client.putMapping(request)) - .then(); - } - if (justUpdateMapping) { - return Mono.empty(); - } - return doCreateIndex(metadata); - }); - } - - protected Mono doLoadIndexMetadata(String _index) { - String index = wrapIndex(_index); - return client.getMapping(new GetMappingsRequest().indices(index)) - .flatMap(resp -> Mono.justOrEmpty(convertMetadata(index, resp.mappings().get(index)))); - } - - - protected CreateIndexRequest createIndexRequest(ElasticSearchIndexMetadata metadata) { - CreateIndexRequest request = new CreateIndexRequest(wrapIndex(metadata.getIndex())); - request.settings(properties.toSettings()); - Map mappingConfig = new HashMap<>(); - mappingConfig.put("properties", createElasticProperties(metadata.getProperties())); - mappingConfig.put("dynamic_templates", createDynamicTemplates()); - if (client.serverVersion().after(Version.V_7_0_0)) { - request.mapping(mappingConfig); - } else { - request.mapping(Collections.singletonMap("_doc", mappingConfig)); - } - return request; - } - - private PutMappingRequest createPutMappingRequest(ElasticSearchIndexMetadata metadata, ElasticSearchIndexMetadata ignore) { - Map properties = createElasticProperties(metadata.getProperties()); - Map ignoreProperties = createElasticProperties(ignore.getProperties()); - for (Map.Entry en : ignoreProperties.entrySet()) { - log.trace("ignore update index [{}] mapping property:{},{}", wrapIndex(metadata.getIndex()), en.getKey(), en - .getValue()); - properties.remove(en.getKey()); - } - if (properties.isEmpty()) { - log.debug("ignore update index [{}] mapping", wrapIndex(metadata.getIndex())); - return null; - } - Map mappingConfig = new HashMap<>(); - PutMappingRequest request = new PutMappingRequest(wrapIndex(metadata.getIndex())); - List allProperties = new ArrayList<>(); - allProperties.addAll(metadata.getProperties()); - allProperties.addAll(ignore.getProperties()); - - mappingConfig.put("properties", createElasticProperties(allProperties)); - request.source(mappingConfig); - return request; - } - - protected Map createElasticProperties(List metadata) { - if (metadata == null) { - return new HashMap<>(); - } - return metadata - .stream() - .collect(Collectors.toMap(PropertyMetadata::getId, - prop -> this.createElasticProperty(prop.getValueType()), (a, v) -> a)); - } - - protected Map createElasticProperty(DataType type) { - Map property = new HashMap<>(); - if (type instanceof DateTimeType) { - property.put("type", "date"); - property.put("format", ElasticDateFormat.getFormat( - ElasticDateFormat.epoch_millis, - ElasticDateFormat.strict_date_hour_minute_second, - ElasticDateFormat.strict_date_time, - ElasticDateFormat.strict_date) - ); - } else if (type instanceof DoubleType) { - property.put("type", "double"); - } else if (type instanceof LongType) { - property.put("type", "long"); - } else if (type instanceof IntType) { - property.put("type", "integer"); - } else if (type instanceof FloatType) { - property.put("type", "float"); - } else if (type instanceof BooleanType) { - property.put("type", "boolean"); - } else if (type instanceof GeoType) { - property.put("type", "geo_point"); - } else if (type instanceof GeoShapeType) { - property.put("type", "geo_shape"); - } else if (type instanceof ArrayType) { - ArrayType arrayType = ((ArrayType) type); - return createElasticProperty(arrayType.getElementType()); - } else if (type instanceof ObjectType) { - property.put("type", "nested"); - ObjectType objectType = ((ObjectType) type); - if (!CollectionUtils.isEmpty(objectType.getProperties())) { - property.put("properties", createElasticProperties(objectType.getProperties())); - } - } else { - property.put("type", "keyword"); - property.put("ignore_above", 512); - } - return property; - } - - protected ElasticSearchIndexMetadata convertMetadata(String index, MappingMetadata metadata) { - MappingMetadata mappingMetadata; - Object properties = null; - Map metaData = metadata.getSourceAsMap(); - - if (metaData.containsKey("properties")) { - Object res = metaData.get("properties"); - if (res instanceof Map) { - properties = res; - } else if (res instanceof MappingMetadata) { - mappingMetadata = ((MappingMetadata) res); - properties = mappingMetadata.sourceAsMap(); - } else if (res instanceof CompressedXContent) { - mappingMetadata = new MappingMetadata(((CompressedXContent) res)); - properties = mappingMetadata.sourceAsMap(); - } else { - throw new UnsupportedOperationException("unsupported index metadata" + metaData); - } - - } else { - Object res = metaData.get("_doc"); - if (res instanceof MappingMetadata) { - mappingMetadata = ((MappingMetadata) res); - } else if (res instanceof CompressedXContent) { - mappingMetadata = new MappingMetadata(((CompressedXContent) res)); - } else { - throw new UnsupportedOperationException("unsupported index metadata" + metaData); - } - properties = mappingMetadata.getSourceAsMap().get("properties"); - } - if (properties == null) { - throw new UnsupportedOperationException("unsupported index metadata" + metaData); - } - - return new DefaultElasticSearchIndexMetadata(index, convertProperties(properties)); - } - - @SuppressWarnings("all") - protected List convertProperties(Object properties) { - if (properties == null) { - return new ArrayList<>(); - } - return ((Map>) properties) - .entrySet() - .stream() - .map(entry -> convertProperty(entry.getKey(), entry.getValue())) - .collect(Collectors.toList()); - } - - private PropertyMetadata convertProperty(String property, Map map) { - String type = String.valueOf(map.get("type")); - SimplePropertyMetadata metadata = new SimplePropertyMetadata(); - metadata.setId(property); - metadata.setName(property); - ElasticPropertyType elasticPropertyType = ElasticPropertyType.of(type); - if (null != elasticPropertyType) { - DataType dataType = elasticPropertyType.getType(); - if ((elasticPropertyType == ElasticPropertyType.OBJECT - || elasticPropertyType == ElasticPropertyType.NESTED) - && dataType instanceof ObjectType) { - @SuppressWarnings("all") - Map nestProperties = (Map) map.get("properties"); - if (null != nestProperties) { - ObjectType objectType = ((ObjectType) dataType); - objectType.setProperties(convertProperties(nestProperties)); - } - } - metadata.setValueType(dataType); - } else { - metadata.setValueType(StringType.GLOBAL); - } - return metadata; - } - - protected List createDynamicTemplates() { - List> maps = new ArrayList<>(); - { - Map config = new HashMap<>(); - config.put("match_mapping_type", "string"); - config.put("mapping", createElasticProperty(StringType.GLOBAL)); - maps.add(Collections.singletonMap("string_fields", config)); - } - { - Map config = new HashMap<>(); - config.put("match_mapping_type", "date"); - config.put("mapping", createElasticProperty(DateTimeType.GLOBAL)); - maps.add(Collections.singletonMap("date_fields", config)); - } - - return maps; - } -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/parser/DefaultLinkTypeParser.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/parser/DefaultLinkTypeParser.java deleted file mode 100644 index 7dfa8ef9..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/parser/DefaultLinkTypeParser.java +++ /dev/null @@ -1,73 +0,0 @@ -package org.jetlinks.community.elastic.search.parser; - -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.hswebframework.ezorm.core.param.Term; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; - -import java.util.List; -import java.util.function.Consumer; - -/** - * @author bsetfeng - * @since 1.0 - **/ -@Component -public class DefaultLinkTypeParser implements LinkTypeParser { - - private final TermTypeParser parser = new DefaultTermTypeParser(); - - - @Override - public void process(List terms, - Consumer consumer, - BoolQueryBuilder queryBuilders) { - - if (CollectionUtils.isEmpty(terms)) { - return; - } - for (TermsHandler.TermGroup group : TermsHandler.groupTerms(terms)) { - if (group.type == Term.Type.or) { - for (Term groupTerm : group.getTerms()) { - handleOr(queryBuilders, groupTerm, consumer); - } - } else { - BoolQueryBuilder andQuery = QueryBuilders.boolQuery(); - for (Term groupTerm : group.getTerms()) { - handleAnd(andQuery, groupTerm, consumer); - } - if (!CollectionUtils.isEmpty(andQuery.must())){ - queryBuilders.should(andQuery); - } - } - } - } - - private void handleOr(BoolQueryBuilder queryBuilders, - Term term, - Consumer consumer) { - consumer.accept(term); - if (term.getTerms().isEmpty() && term.getValue() != null) { - parser.process(() -> term, queryBuilders::should); - } else if (!term.getTerms().isEmpty()){ - BoolQueryBuilder nextQuery = QueryBuilders.boolQuery(); - process(term.getTerms(), consumer, nextQuery); - queryBuilders.should(nextQuery); - } - } - - private void handleAnd(BoolQueryBuilder queryBuilders, - Term term, - Consumer consumer) { - consumer.accept(term); - if (term.getTerms().isEmpty() && term.getValue() != null) { - parser.process(() -> term, queryBuilders::must); - } else if (!term.getTerms().isEmpty()){ - BoolQueryBuilder nextQuery = QueryBuilders.boolQuery(); - process(term.getTerms(), consumer, nextQuery); - queryBuilders.must(nextQuery); - } - } - -} \ No newline at end of file diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/parser/DefaultTermTypeParser.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/parser/DefaultTermTypeParser.java deleted file mode 100644 index 84800ef5..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/parser/DefaultTermTypeParser.java +++ /dev/null @@ -1,39 +0,0 @@ -package org.jetlinks.community.elastic.search.parser; - -import org.apache.lucene.search.join.ScoreMode; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.NestedQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.hswebframework.ezorm.core.param.Term; -import org.jetlinks.community.elastic.search.enums.TermTypeEnum; - -import java.util.function.Function; -import java.util.function.Supplier; - -/** - * @author bsetfeng - * @since 1.0 - **/ -public class DefaultTermTypeParser implements TermTypeParser { - - - @Override - public void process(Supplier termSupplier, Function function) { - function.apply(queryBuilder(termSupplier.get())); - } - - - private QueryBuilder queryBuilder(Term term) { - return TermTypeEnum.of(term.getTermType().trim()) - .map(e -> createQueryBuilder(e,term)) - .orElse(QueryBuilders.boolQuery()); - } - - static QueryBuilder createQueryBuilder(TermTypeEnum linkTypeEnum, Term term) { - if (term.getColumn().contains(".")) { - return new NestedQueryBuilder(term.getColumn().split("[.]")[0], linkTypeEnum.process(term), ScoreMode.Max); - } - return linkTypeEnum.process(term); - } -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/parser/LinkTypeParser.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/parser/LinkTypeParser.java deleted file mode 100644 index cc92201b..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/parser/LinkTypeParser.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.jetlinks.community.elastic.search.parser; - -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.hswebframework.ezorm.core.param.Term; - -import java.util.List; -import java.util.function.Consumer; - -/** - * @version 1.0 - **/ -public interface LinkTypeParser { - - void process(List terms, Consumer consumer, BoolQueryBuilder queryBuilders); -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/parser/TermTypeParser.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/parser/TermTypeParser.java deleted file mode 100644 index 866d461a..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/parser/TermTypeParser.java +++ /dev/null @@ -1,17 +0,0 @@ -package org.jetlinks.community.elastic.search.parser; - -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.hswebframework.ezorm.core.param.Term; - -import java.util.function.Function; -import java.util.function.Supplier; - -/** - * @version 1.0 - **/ -public interface TermTypeParser { - - void process(Supplier termSupplier, Function function); - -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/parser/TermsHandler.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/parser/TermsHandler.java deleted file mode 100644 index ffac5fab..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/parser/TermsHandler.java +++ /dev/null @@ -1,71 +0,0 @@ -package org.jetlinks.community.elastic.search.parser; - -import lombok.Getter; -import lombok.Setter; -import org.hswebframework.ezorm.core.param.Term; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -/** - * 根据terms条件的type分组 - * 如：条件 where a = 1 or b = 2 and c = 3 or d = 4 or e = 5 and f = 6 - * 分组为 - * (a = 1) - * OR - * (b = 2 AND c = 3) - * OR - * (d = 4) - * OR - * (e = 5 AND f = 6) - * - * @author bestfeng - */ -public class TermsHandler { - - public static Set groupTerms(List terms) { - Set groups = new HashSet<>(); - TermGroup currentGroup = null; - - for (int i = 0; i < terms.size(); i++) { - Term currentTerm = terms.get(i); - Term nextTerm = (i + 1 < terms.size()) ? terms.get(i + 1) : null; - - if (currentTerm.getType() == Term.Type.or){ - if (currentGroup == null || currentGroup.type == Term.Type.and){ - currentGroup = new TermGroup(Term.Type.or); - } - // 如果下一个Term为"AND"，创建一个新分组 - if (nextTerm != null && nextTerm.getType() == Term.Type.and){ - currentGroup = new TermGroup(Term.Type.and); - } - }else { - if (currentGroup == null){ - currentGroup = new TermGroup(Term.Type.and); - } - } - currentGroup.addTerm(currentTerm); - groups.add(currentGroup); - } - return groups; - } - - @Getter - @Setter - public static class TermGroup { - public TermGroup(Term.Type type) { - this.type = type; - this.terms = new ArrayList<>(); - } - - List terms; - - Term.Type type; - public void addTerm(Term term) { - terms.add(term); - } - } - -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/ElasticSearchService.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/ElasticSearchService.java deleted file mode 100644 index 8a51fee0..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/ElasticSearchService.java +++ /dev/null @@ -1,99 +0,0 @@ -package org.jetlinks.community.elastic.search.service; - -import org.hswebframework.ezorm.core.param.QueryParam; -import org.hswebframework.web.api.crud.entity.PagerResult; -import org.hswebframework.web.bean.FastBeanCopier; -import org.jetlinks.community.elastic.search.index.ElasticIndex; -import org.reactivestreams.Publisher; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; - -import java.util.Collection; -import java.util.Map; -import java.util.function.Function; - -/** - * ES数据库业务操作类 - * - * @author zhouhao - */ -public interface ElasticSearchService { - - default Mono> queryPager(String index, QueryParam queryParam, Function, T> mapper) { - return queryPager(new String[]{index}, queryParam, mapper); - } - - Mono> queryPager(String[] index, QueryParam queryParam, Function, T> mapper); - - Flux query(String index, QueryParam queryParam, Function, T> mapper); - - Flux query(String[] index, QueryParam queryParam, Function, T> mapper); - - default Flux multiQuery(String index, Collection queryParam, Function, T> mapper) { - return multiQuery(new String[]{index}, queryParam, mapper); - } - - Flux multiQuery(String[] index, Collection queryParam, Function, T> mapper); - - default Mono count(String index, QueryParam queryParam) { - return count(new String[]{index}, queryParam); - } - - Mono count(String[] index, QueryParam queryParam); - - - Mono delete(String index, QueryParam queryParam); - - Mono commit(String index, T payload); - - Mono commit(String index, Collection payload); - - Mono commit(String index, Publisher data); - - Mono save(String index, T payload); - - Mono save(String index, Collection payload); - - Mono save(String index, Publisher data); - - default Flux query(String index, QueryParam queryParam, Class type) { - return query(index, queryParam, map -> FastBeanCopier.copy(map, type)); - } - - default Mono> queryPager(String index, QueryParam queryParam, Class type) { - return queryPager(index, queryParam, map -> FastBeanCopier.copy(map, type)); - } - - default Mono> queryPager(ElasticIndex index, QueryParam queryParam, Class type) { - return queryPager(index.getIndex(), queryParam, map -> FastBeanCopier.copy(map, type)); - } - - default Mono> queryPager(ElasticIndex index, QueryParam queryParam, Function, T> mapper) { - return queryPager(index.getIndex(), queryParam, mapper); - } - - default Flux query(ElasticIndex index, QueryParam queryParam, Class type) { - return query(index.getIndex(), queryParam, map -> FastBeanCopier.copy(map, type)); - } - - default Flux query(ElasticIndex index, QueryParam queryParam, Function, T> mapper) { - return this.query(index.getIndex(), queryParam, mapper); - } - - default Mono count(ElasticIndex index, QueryParam data) { - return this.count(index.getIndex(), data); - } - - default Mono commit(ElasticIndex index, T data) { - return this.commit(index.getIndex(), data); - } - - default Mono commit(ElasticIndex index, Collection data) { - return this.commit(index.getIndex(), data); - } - - default Mono commit(ElasticIndex index, Publisher data) { - return this.commit(index.getIndex(), data); - } - -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/reactive/AggType.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/reactive/AggType.java deleted file mode 100755 index 409ba07c..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/reactive/AggType.java +++ /dev/null @@ -1,101 +0,0 @@ -package org.jetlinks.community.elastic.search.service.reactive; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import org.elasticsearch.search.aggregations.AggregationBuilder; -import org.elasticsearch.search.aggregations.AggregationBuilders; - -@AllArgsConstructor -public enum AggType { - - AVG("平均") { - @Override - public AggregationBuilder aggregationBuilder(String name, String filed) { - return AggregationBuilders.avg(name).field(filed).missing(0); - } - - }, - MAX("最大") { - @Override - public AggregationBuilder aggregationBuilder(String name, String filed) { - return AggregationBuilders.max(name).field(filed).missing(0); - } - }, - MEDIAN("中间值") { - @Override - public AggregationBuilder aggregationBuilder(String name, String filed) { - return AggregationBuilders.medianAbsoluteDeviation(name).field(filed).missing(0); - } - }, - STDDEV("标准差") { - @Override - public AggregationBuilder aggregationBuilder(String name, String filed) { - return AggregationBuilders.extendedStats(name) - .field(filed) - .missing(0); - } - }, - COUNT("非空值计数") { - @Override - public AggregationBuilder aggregationBuilder(String name, String filed) { - return AggregationBuilders.count(name).field(filed).missing(0); - } - - }, - DISTINCT_COUNT("去重计数") { - @Override - public AggregationBuilder aggregationBuilder(String name, String filed) { - return AggregationBuilders - .cardinality(name) - .field(filed) - .missing(0); - } - - }, - MIN("最小") { - @Override - public AggregationBuilder aggregationBuilder(String name, String filed) { - return AggregationBuilders.min(name).field(filed).missing(0); - } - }, - FIRST("第一条数据") { - @Override - public AggregationBuilder aggregationBuilder(String name, String filed) { - return AggregationBuilders.topHits(name).size(1); - } - }, - TOP("第N条数据") { - @Override - public AggregationBuilder aggregationBuilder(String name, String filed) { - return AggregationBuilders.topHits(name); - } - }, - SUM("总数") { - @Override - public AggregationBuilder aggregationBuilder(String name, String filed) { - return AggregationBuilders.sum(name).field(filed).missing(0); - } - }, - STATS("统计汇总") { - @Override - public AggregationBuilder aggregationBuilder(String name, String filed) { - return AggregationBuilders.stats(name).field(filed).missing(0); - } - - }; - - @Getter - private final String text; - - public abstract AggregationBuilder aggregationBuilder(String name, String filed); - - public static AggType of(String name) { - for (AggType type : AggType.values()) { - if (type.name().equalsIgnoreCase(name)) { - return type; - } - } - throw new UnsupportedOperationException("不支持的聚合类型：" + name); - } - -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/reactive/DefaultReactiveElasticsearchClient.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/reactive/DefaultReactiveElasticsearchClient.java deleted file mode 100755 index 56a0bd44..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/reactive/DefaultReactiveElasticsearchClient.java +++ /dev/null @@ -1,1599 +0,0 @@ -package org.jetlinks.community.elastic.search.service.reactive; - -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONObject; -import lombok.Generated; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPut; -import org.apache.http.entity.ByteArrayEntity; -import org.apache.http.entity.ContentType; -import org.apache.http.util.EntityUtils; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.ElasticsearchStatusException; -import org.elasticsearch.Version; -import org.elasticsearch.action.ActionRequest; -import org.elasticsearch.action.DocWriteRequest; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; -import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; -import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; -import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; -import org.elasticsearch.action.admin.indices.flush.FlushRequest; -import org.elasticsearch.action.admin.indices.flush.FlushResponse; -import org.elasticsearch.action.admin.indices.get.GetIndexRequest; -import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; -import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; -import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; -import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; -import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; -import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; -import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; -import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest; -import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesRequest; -import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse; -import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; -import org.elasticsearch.action.bulk.BulkRequest; -import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.delete.DeleteRequest; -import org.elasticsearch.action.delete.DeleteResponse; -import org.elasticsearch.action.get.*; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.index.IndexResponse; -import org.elasticsearch.action.main.MainRequest; -import org.elasticsearch.action.main.MainResponse; -import org.elasticsearch.action.search.*; -import org.elasticsearch.action.support.ActiveShardCount; -import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.action.support.WriteRequest; -import org.elasticsearch.action.support.master.AcknowledgedResponse; -import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.action.update.UpdateResponse; -import org.elasticsearch.client.GetAliasesResponse; -import org.elasticsearch.client.Request; -import org.elasticsearch.client.indices.GetFieldMappingsRequest; -import org.elasticsearch.client.indices.GetFieldMappingsResponse; -import org.elasticsearch.client.indices.GetIndexResponse; -import org.elasticsearch.client.indices.IndexTemplatesExistRequest; -import org.elasticsearch.client.tasks.TaskSubmissionResponse; -import org.elasticsearch.cluster.health.ClusterHealthStatus; -import org.elasticsearch.common.Priority; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.lucene.uid.Versions; -import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.core.TimeValue; -import org.elasticsearch.index.VersionType; -import org.elasticsearch.index.get.GetResult; -import org.elasticsearch.index.reindex.BulkByScrollResponse; -import org.elasticsearch.index.reindex.DeleteByQueryRequest; -import org.elasticsearch.index.reindex.ReindexRequest; -import org.elasticsearch.index.reindex.UpdateByQueryRequest; -import org.elasticsearch.index.seqno.SequenceNumbers; -import org.elasticsearch.rest.BytesRestResponse; -import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.script.mustache.SearchTemplateRequest; -import org.elasticsearch.script.mustache.SearchTemplateResponse; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.SearchHits; -import org.elasticsearch.search.aggregations.Aggregation; -import org.elasticsearch.search.fetch.subphase.FetchSourceContext; -import org.elasticsearch.search.internal.SearchContext; -import org.elasticsearch.search.suggest.Suggest; -import org.elasticsearch.tasks.TaskId; -import org.elasticsearch.xcontent.*; -import org.reactivestreams.Publisher; -import org.springframework.data.elasticsearch.client.ClientLogger; -import org.springframework.data.elasticsearch.client.ElasticsearchHost; -import org.springframework.data.elasticsearch.client.NoReachableHostException; -import org.springframework.data.elasticsearch.client.reactive.HostProvider; -import org.springframework.data.elasticsearch.client.reactive.RequestCreator; -import org.springframework.data.elasticsearch.client.util.NamedXContents; -import org.springframework.data.elasticsearch.client.util.RequestConverters; -import org.springframework.data.elasticsearch.client.util.ScrollState; -import org.springframework.data.elasticsearch.core.ResponseConverter; -import org.springframework.data.elasticsearch.core.query.ByQueryResponse; -import org.springframework.data.util.Lazy; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpMethod; -import org.springframework.http.MediaType; -import org.springframework.lang.Nullable; -import org.springframework.util.Assert; -import org.springframework.util.ObjectUtils; -import org.springframework.util.ReflectionUtils; -import org.springframework.web.reactive.function.BodyExtractors; -import org.springframework.web.reactive.function.client.ClientRequest; -import org.springframework.web.reactive.function.client.ClientResponse; -import org.springframework.web.reactive.function.client.WebClient; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; -import reactor.function.Function3; - -import javax.annotation.Nonnull; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.lang.reflect.Method; -import java.net.ConnectException; -import java.nio.charset.StandardCharsets; -import java.time.Duration; -import java.util.Collection; -import java.util.Locale; -import java.util.Map; -import java.util.StringJoiner; -import java.util.function.Function; -import java.util.function.Supplier; - -import static org.springframework.data.elasticsearch.client.util.RequestConverters.createContentType; - -@Slf4j -@Generated -public class DefaultReactiveElasticsearchClient implements ReactiveElasticsearchClient, - org.springframework.data.elasticsearch.client.reactive.ReactiveElasticsearchClient.Cluster { - private final HostProvider hostProvider; - private final RequestCreator requestCreator; - private Supplier headersSupplier = () -> HttpHeaders.EMPTY; - - /** - * Create a new {@link org.springframework.data.elasticsearch.client.reactive.DefaultReactiveElasticsearchClient} using the given {@link HostProvider} to obtain server - * connections and the given {@link RequestCreator}. - * - * @param hostProvider must not be {@literal null}. - * @param requestCreator must not be {@literal null}. - */ - public DefaultReactiveElasticsearchClient(HostProvider hostProvider, RequestCreator requestCreator) { - - Assert.notNull(hostProvider, "HostProvider must not be null"); - Assert.notNull(requestCreator, "RequestCreator must not be null"); - - this.hostProvider = hostProvider; - this.requestCreator = requestCreator; - version = info().block(Duration.ofSeconds(10)).getVersion(); - } - - public void setHeadersSupplier(Supplier headersSupplier) { - - Assert.notNull(headersSupplier, "headersSupplier must not be null"); - - this.headersSupplier = headersSupplier; - } - - /* - * (non-Javadoc) - * @see org.springframework.data.elasticsearch.client.reactive.ReactiveElasticsearchClient#ping(org.springframework.http.HttpHeaders) - */ - @Override - public Mono ping(HttpHeaders headers) { - - return sendRequest(new MainRequest(), requestCreator.ping(), RawActionResponse.class, headers) - .flatMap(response -> response.releaseBody().thenReturn(response.statusCode().is2xxSuccessful())) - .onErrorResume(NoReachableHostException.class, error -> Mono.just(false)) - .next(); - } - - /* - * (non-Javadoc) - * @see org.springframework.data.elasticsearch.client.reactive.ReactiveElasticsearchClient#info(org.springframework.http.HttpHeaders) - */ - @Override - public Mono info(HttpHeaders headers) { - - return sendRequest(new MainRequest(), requestCreator.info(), MainResponse.class, headers) // - .next(); - } - - /* - * (non-Javadoc) - * @see org.springframework.data.elasticsearch.client.reactive.ReactiveElasticsearchClient#get(org.springframework.http.HttpHeaders, org.elasticsearch.action.get.GetRequest) - */ - @Override - public Mono get(HttpHeaders headers, GetRequest getRequest) { - - return sendRequest(getRequest, requestCreator.get(), GetResponse.class, headers) // - .filter(GetResponse::isExists) // - .map(DefaultReactiveElasticsearchClient::getResponseToGetResult) // - .next(); - } - - /* - * (non-Javadoc) - * @see org.springframework.data.elasticsearch.client.reactive.ReactiveElasticsearchClient#multiGet(org.springframework.http.HttpHeaders, org.elasticsearch.action.get.MultiGetRequest) - */ - @Override - public Flux multiGet(HttpHeaders headers, MultiGetRequest multiGetRequest) { - - return sendRequest(multiGetRequest, requestCreator.multiGet(), MultiGetResponse.class, headers) - .map(MultiGetResponse::getResponses) // - .flatMap(Flux::fromArray); // - } - - /* - * (non-Javadoc) - * @see org.springframework.data.elasticsearch.client.reactive.ReactiveElasticsearchClient#exists(org.springframework.http.HttpHeaders, org.elasticsearch.action.get.GetRequest) - */ - @Override - public Mono exists(HttpHeaders headers, GetRequest getRequest) { - - return sendRequest(getRequest, requestCreator.exists(), RawActionResponse.class, headers) - .flatMap(response -> response.releaseBody().thenReturn(response - .statusCode() - .is2xxSuccessful())) - .next(); - } - - /* - * (non-Javadoc) - * @see org.springframework.data.elasticsearch.client.reactive.ReactiveElasticsearchClient#ping(org.springframework.http.HttpHeaders, org.elasticsearch.action.index.IndexRequest) - */ - @Override - public Mono index(HttpHeaders headers, IndexRequest indexRequest) { - return sendRequest(indexRequest, requestCreator.index(), IndexResponse.class, headers).publishNext(); - } - - /* - * (non-Javadoc) - * @see org.springframework.data.elasticsearch.client.reactive.ReactiveElasticsearchClient#indices() - */ - @Override - public Indices indices() { - return this; - } - - @Override - public Cluster cluster() { - return this; - } - - /* - * (non-Javadoc) - * @see org.springframework.data.elasticsearch.client.reactive.ReactiveElasticsearchClient#ping(org.springframework.http.HttpHeaders, org.elasticsearch.action.update.UpdateRequest) - */ - @Override - public Mono update(HttpHeaders headers, UpdateRequest updateRequest) { - return sendRequest(updateRequest, requestCreator.update(), UpdateResponse.class, headers).publishNext(); - } - - /* - * (non-Javadoc) - * @see org.springframework.data.elasticsearch.client.reactive.ReactiveElasticsearchClient#ping(org.springframework.http.HttpHeaders, org.elasticsearch.action.delete.DeleteRequest) - */ - @Override - public Mono delete(HttpHeaders headers, DeleteRequest deleteRequest) { - - return sendRequest(deleteRequest, requestCreator.delete(), DeleteResponse.class, headers) // - .publishNext(); - } - - /* - * (non-Javadoc) - * @see org.springframework.data.elasticsearch.client.reactive.ReactiveElasticsearchClient#count(org.springframework.http.HttpHeaders, org.elasticsearch.action.search.SearchRequest) - */ - @Override - public Mono count(HttpHeaders headers, SearchRequest searchRequest) { - searchRequest.source().trackTotalHits(true); - searchRequest.source().size(0); - searchRequest.source().fetchSource(false); - return sendRequest(searchRequest, this::buildSearchRequest, SearchResponse.class, headers) - .map(SearchResponse::getHits) - .map(searchHits -> searchHits.getTotalHits().value) - .next(); - } - - @Override - public Flux searchTemplate(HttpHeaders headers, SearchTemplateRequest searchTemplateRequest) { - return sendRequest(searchTemplateRequest, requestCreator.searchTemplate(), SearchTemplateResponse.class, headers) - .map(response -> response.getResponse().getHits()).flatMap(Flux::fromIterable); - } - - protected Request buildSearchRequest(SearchRequest request) { - //兼容6.x版本es - if (version.before(Version.V_7_0_0) && request - .source() - .trackTotalHitsUpTo() != SearchContext.TRACK_TOTAL_HITS_DISABLED) { - Request req = requestCreator.search().apply(request); - JSONObject json = JSON.parseObject(requestBodyToString(req)); - json.put("track_total_hits", true); - req.setJsonEntity(json.toJSONString()); - return req; - } - return requestCreator.search().apply(request); - } - - /* - * (non-Javadoc) - * @see org.springframework.data.elasticsearch.client.reactive.ReactiveElasticsearchClient#ping(org.springframework.http.HttpHeaders, org.elasticsearch.action.search.SearchRequest) - */ - @Override - public Flux search(HttpHeaders headers, SearchRequest searchRequest) { - - return sendRequest(searchRequest, this::buildSearchRequest, SearchResponse.class, headers) // - .map(SearchResponse::getHits) // - .flatMap(Flux::fromIterable); - } - - @Override - public Mono searchForResponse(HttpHeaders headers, SearchRequest searchRequest) { - return sendRequest(searchRequest, requestCreator.search(), SearchResponse.class, headers).next(); - } - - @Override - public Flux suggest(HttpHeaders headers, SearchRequest searchRequest) { - return sendRequest(searchRequest, requestCreator.search(), SearchResponse.class, headers) // - .map(SearchResponse::getSuggest); - } - - /* - * (non-Javadoc) - * @see org.springframework.data.elasticsearch.client.reactive.ReactiveElasticsearchClient#aggregate(org.springframework.http.HttpHeaders, org.elasticsearch.action.search.SearchRequest) - */ - @Override - public Flux aggregate(HttpHeaders headers, SearchRequest searchRequest) { - - Assert.notNull(headers, "headers must not be null"); - Assert.notNull(searchRequest, "searchRequest must not be null"); - - searchRequest.source().size(0); - searchRequest.source().trackTotalHits(false); - - return sendRequest(searchRequest, this::buildSearchRequest, SearchResponse.class, headers) // - .map(SearchResponse::getAggregations) // - .flatMap(Flux::fromIterable); - } - - @Override@Nonnull - public Flux scroll(@Nonnull HttpHeaders headers, SearchRequest searchRequest) { - - TimeValue scrollTimeout = searchRequest.scroll() != null ? searchRequest.scroll().keepAlive() - : TimeValue.timeValueMinutes(1); - - if (searchRequest.scroll() == null) { - searchRequest.scroll(scrollTimeout); - } - - return Flux - .usingWhen(Mono.fromSupplier(ScrollState::new), - state -> this - .sendRequest(searchRequest, requestCreator.search(), SearchResponse.class, headers) - .expand(searchResponse -> { - - state.updateScrollId(searchResponse.getScrollId()); - if (isEmpty(searchResponse.getHits())) { - return Mono.empty(); - } - - return this - .sendRequest(new SearchScrollRequest(searchResponse.getScrollId()).scroll(scrollTimeout), - requestCreator.scroll(), - SearchResponse.class, - headers); - - }), - state -> cleanupScroll(headers, state), - (state, ex) -> cleanupScroll(headers, state), - state -> cleanupScroll(headers, state)) - .filter(it -> !isEmpty(it.getHits())) - .map(SearchResponse::getHits) - .flatMapIterable(Function.identity()); - } - private static boolean isEmpty(@Nullable SearchHits hits) { - return hits != null && hits.getHits() != null && hits.getHits().length == 0; - } - - private Publisher cleanupScroll(HttpHeaders headers, ScrollState state) { - - if (state.getScrollIds().isEmpty()) { - return Mono.empty(); - } - - ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); - clearScrollRequest.scrollIds(state.getScrollIds()); - - // just send the request, resources get cleaned up anyways after scrollTimeout has been reached. - return sendRequest(clearScrollRequest, requestCreator.clearScroll(), ClearScrollResponse.class, headers); - } - - /* - * (non-Javadoc) - * @see org.springframework.data.elasticsearch.client.reactive.ReactiveElasticsearchClient#ping(org.springframework.http.HttpHeaders, org.elasticsearch.index.reindex.DeleteByQueryRequest) - */ - @Override - public Mono deleteBy(HttpHeaders headers, DeleteByQueryRequest deleteRequest) { - - return sendRequest(deleteRequest, requestCreator.deleteByQuery(), BulkByScrollResponse.class, headers) // - .publishNext(); - } - - @Override - public Mono updateBy(HttpHeaders headers, UpdateByQueryRequest updateRequest) { - return sendRequest(updateRequest, requestCreator.updateByQuery(), BulkByScrollResponse.class, headers) - .next() - .map(ResponseConverter::byQueryResponseOf); - } - - static XContentType enforceSameContentType(IndexRequest indexRequest, @Nullable XContentType xContentType) { - XContentType requestContentType = indexRequest.getContentType(); - if (requestContentType != XContentType.JSON && requestContentType != XContentType.SMILE) { - throw new IllegalArgumentException("Unsupported content-type found for request with content-type [" - + requestContentType + "], only JSON and SMILE are supported"); - } - if (xContentType == null) { - return requestContentType; - } - if (requestContentType != xContentType) { - throw new IllegalArgumentException("Mismatching content-type found for request with content-type [" - + requestContentType + "], previous requests have content-type [" + xContentType + ']'); - } - return xContentType; - } - - @SneakyThrows - Request convertBulk(BulkRequest bulkRequest) { - Request request = new Request(HttpMethod.POST.name(), "/_bulk"); - - Params parameters = new Params(request); - parameters.withTimeout(bulkRequest.timeout()); - parameters.withRefreshPolicy(bulkRequest.getRefreshPolicy()); - - // parameters.withPipeline(bulkRequest.pipeline()); - // parameters.withRouting(bulkRequest.routing()); - - // Bulk API only supports newline delimited JSON or Smile. Before executing - // the bulk, we need to check that all requests have the same content-type - // and this content-type is supported by the Bulk API. - XContentType bulkContentType = null; - for (int i = 0; i < bulkRequest.numberOfActions(); i++) { - DocWriteRequest action = bulkRequest.requests().get(i); - - DocWriteRequest.OpType opType = action.opType(); - if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) { - bulkContentType = enforceSameContentType((IndexRequest) action, bulkContentType); - - } else if (opType == DocWriteRequest.OpType.UPDATE) { - UpdateRequest updateRequest = (UpdateRequest) action; - if (updateRequest.doc() != null) { - bulkContentType = enforceSameContentType(updateRequest.doc(), bulkContentType); - } - if (updateRequest.upsertRequest() != null) { - bulkContentType = enforceSameContentType(updateRequest.upsertRequest(), bulkContentType); - } - } - } - - if (bulkContentType == null) { - bulkContentType = XContentType.JSON; - } - - final byte separator = bulkContentType.xContent().streamSeparator(); - final ContentType requestContentType = createContentType(bulkContentType); - - ByteArrayOutputStream content = new ByteArrayOutputStream(); - for (DocWriteRequest action : bulkRequest.requests()) { - DocWriteRequest.OpType opType = action.opType(); - - try (XContentBuilder metadata = XContentBuilder.builder(bulkContentType.xContent())) { - metadata.startObject(); - { - metadata.startObject(opType.getLowercase()); - if (Strings.hasLength(action.index())) { - metadata.field("_index", action.index()); - } - if (Strings.hasLength(action.type())) { - metadata.field("_type", action.type()); - } - if (Strings.hasLength(action.id())) { - metadata.field("_id", action.id()); - } - if (Strings.hasLength(action.routing())) { - metadata.field("routing", action.routing()); - } - if (action.version() != Versions.MATCH_ANY) { - metadata.field("version", action.version()); - } - - VersionType versionType = action.versionType(); - if (versionType != VersionType.INTERNAL) { - if (versionType == VersionType.EXTERNAL) { - metadata.field("version_type", "external"); - } else if (versionType == VersionType.EXTERNAL_GTE) { - metadata.field("version_type", "external_gte"); - } - } - - if (action.ifSeqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO) { - metadata.field("if_seq_no", action.ifSeqNo()); - metadata.field("if_primary_term", action.ifPrimaryTerm()); - } - - if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) { - IndexRequest indexRequest = (IndexRequest) action; - if (Strings.hasLength(indexRequest.getPipeline())) { - metadata.field("pipeline", indexRequest.getPipeline()); - } - } else if (opType == DocWriteRequest.OpType.UPDATE) { - UpdateRequest updateRequest = (UpdateRequest) action; - if (updateRequest.retryOnConflict() > 0) { - metadata.field("retry_on_conflict", updateRequest.retryOnConflict()); - } - if (updateRequest.fetchSource() != null) { - metadata.field("_source", updateRequest.fetchSource()); - } - } - metadata.endObject(); - } - metadata.endObject(); - - BytesRef metadataSource = BytesReference.bytes(metadata).toBytesRef(); - content.write(metadataSource.bytes, metadataSource.offset, metadataSource.length); - content.write(separator); - } - - BytesRef source = null; - if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) { - IndexRequest indexRequest = (IndexRequest) action; - BytesReference indexSource = indexRequest.source(); - XContentType indexXContentType = indexRequest.getContentType(); - - try (XContentParser parser = XContentHelper.createParser( - /* - * EMPTY and THROW are fine here because we just call - * copyCurrentStructure which doesn't touch the - * registry or deprecation. - */ - NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, indexSource, - indexXContentType)) { - try (XContentBuilder builder = XContentBuilder.builder(bulkContentType.xContent())) { - builder.copyCurrentStructure(parser); - source = BytesReference.bytes(builder).toBytesRef(); - } - } - } else if (opType == DocWriteRequest.OpType.UPDATE) { - source = XContentHelper.toXContent((UpdateRequest) action, bulkContentType, false).toBytesRef(); - } - - if (source != null) { - content.write(source.bytes, source.offset, source.length); - content.write(separator); - } - } - request.setEntity(new ByteArrayEntity(content.toByteArray(), 0, content.size(), requestContentType)); - return request; - } - - - /* - * (non-Javadoc) - * @see org.springframework.data.elasticsearch.client.reactive.ReactiveElasticsearchClient#bulk(org.springframework.http.HttpHeaders, org.elasticsearch.action.bulk.BulkRequest) - */ - @Override - public Mono bulk(HttpHeaders headers, BulkRequest bulkRequest) { - return sendRequest(bulkRequest, this::convertBulk, BulkResponse.class, headers) // - .publishNext(); - } - - @Override - public Mono reindex(HttpHeaders headers, ReindexRequest reindexRequest) { - return sendRequest(reindexRequest, requestCreator.reindex(), BulkByScrollResponse.class, headers).next(); - } - - @Override - public Mono submitReindex(HttpHeaders headers, ReindexRequest reindexRequest) { - return sendRequest(reindexRequest, requestCreator.submitReindex(), TaskSubmissionResponse.class, headers).next() - .map(TaskSubmissionResponse::getTask); - } - - // --> INDICES - - /* - * (non-Javadoc) - * @see org.springframework.data.elasticsearch.client.reactive.ReactiveElasticsearchClient.Indices#existsIndex(org.springframework.http.HttpHeaders, org.elasticsearch.action.admin.indices.get.GetIndexRequest) - */ - @Override - public Mono existsIndex(HttpHeaders headers, GetIndexRequest request) { - return sendRequest(request, requestCreator.indexExists() - , RawActionResponse.class, headers) - .flatMap(response -> response - .releaseBody() - .thenReturn(response - .statusCode() - .is2xxSuccessful())) - .onErrorReturn(false) - .next(); - } - - @Override - public Mono existsIndex(HttpHeaders headers, org.elasticsearch.client.indices.GetIndexRequest getIndexRequest) { - return sendRequest(getIndexRequest, requestCreator.indexExistsRequest(), RawActionResponse.class, headers) - .flatMap(response -> response - .releaseBody() - .thenReturn(response - .statusCode() - .is2xxSuccessful())) - .next(); - } - - /* - * (non-Javadoc) - * @see org.springframework.data.elasticsearch.client.reactive.ReactiveElasticsearchClient.Indices#deleteIndex(org.springframework.http.HttpHeaders, org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest) - */ - @Override - public Mono deleteIndex(HttpHeaders headers, DeleteIndexRequest request) { - - return sendRequest(request, requestCreator.indexDelete(), AcknowledgedResponse.class, headers) - .map(AcknowledgedResponse::isAcknowledged) - .next(); - } - - /* - * (non-Javadoc) - * @see org.springframework.data.elasticsearch.client.reactive.ReactiveElasticsearchClient.Indices#createIndex(org.springframework.http.HttpHeaders, org.elasticsearch.action.admin.indices.create.CreateIndexRequest) - */ - @Override - public Mono createIndex(HttpHeaders headers, CreateIndexRequest createIndexRequest) { - - return sendRequest(createIndexRequest, requestCreator.indexCreate().andThen(request -> { - request.addParameter("include_type_name", "true"); - return request; - }), AcknowledgedResponse.class, headers) - .map(AcknowledgedResponse::isAcknowledged) - .next(); - } - - @Override - public Mono createIndex(HttpHeaders headers, org.elasticsearch.client.indices.CreateIndexRequest createIndexRequest) { - return sendRequest(createIndexRequest, requestCreator.createIndexRequest(), AcknowledgedResponse.class, headers) - .map(AcknowledgedResponse::isAcknowledged) - .next(); - } - - /* - * (non-Javadoc) - * @see org.springframework.data.elasticsearch.client.reactive.ReactiveElasticsearchClient.Indices#openIndex(org.springframework.http.HttpHeaders, org.elasticsearch.action.admin.indices.open.OpenIndexRequest) - */ - @Override - public Mono openIndex(HttpHeaders headers, OpenIndexRequest request) { - - return sendRequest(request, requestCreator.indexOpen(), AcknowledgedResponse.class, headers) - .then(); - } - - /* - * (non-Javadoc) - * @see org.springframework.data.elasticsearch.client.reactive.ReactiveElasticsearchClient.Indices#closeIndex(org.springframework.http.HttpHeaders, org.elasticsearch.action.admin.indices.close.CloseIndexRequest) - */ - @Override - public Mono closeIndex(HttpHeaders headers, CloseIndexRequest closeIndexRequest) { - - return sendRequest(closeIndexRequest, requestCreator.indexClose(), AcknowledgedResponse.class, headers) // - .then(); - } - - /* - * (non-Javadoc) - * @see org.springframework.data.elasticsearch.client.reactive.ReactiveElasticsearchClient.Indices#refreshIndex(org.springframework.http.HttpHeaders, org.elasticsearch.action.admin.indices.refresh.RefreshRequest) - */ - @Override - public Mono refreshIndex(HttpHeaders headers, RefreshRequest refreshRequest) { - - return sendRequest(refreshRequest, requestCreator.indexRefresh(), RefreshResponse.class, headers) // - .then(); - } - - @Override - public Mono putMapping(HttpHeaders headers, PutMappingRequest putMappingRequest) { - return sendRequest(putMappingRequest, requestCreator.putMapping(), AcknowledgedResponse.class, headers) - .map(AcknowledgedResponse::isAcknowledged) - .next(); - } - - @Override - public Mono putMapping(HttpHeaders headers, org.elasticsearch.client.indices.PutMappingRequest putMappingRequest) { - - return sendRequest(putMappingRequest, this::createPutMapping, AcknowledgedResponse.class, headers) - .map(AcknowledgedResponse::isAcknowledged) - .next(); - } - - private Request createPutMapping(org.elasticsearch.client.indices.PutMappingRequest putMappingRequest) { - Request request = requestCreator.putMappingRequest().apply(putMappingRequest); - Request newReq = new Request(request.getMethod(), request.getEndpoint()); - - Params params = new Params(newReq) - .withTimeout(putMappingRequest.timeout()) - .withMasterTimeout(putMappingRequest.masterNodeTimeout()); - if (serverVersion().before(Version.V_7_0_0)) { - params.putParam("include_type_name", "false"); - } - newReq.setEntity(request.getEntity()); - return newReq; - } - - /* - * (non-Javadoc) - * @see org.springframework.data.elasticsearch.client.reactive.ReactiveElasticsearchClient.Indices#flushIndex(org.springframework.http.HttpHeaders, org.elasticsearch.action.admin.indices.flush.FlushRequest) - */ - @Override - public Mono flushIndex(HttpHeaders headers, FlushRequest flushRequest) { - - return sendRequest(flushRequest, requestCreator.flushIndex(), FlushResponse.class, headers) // - .then(); - } - - @Override - public Mono getSettings(HttpHeaders headers, GetSettingsRequest getSettingsRequest) { - return sendRequest(getSettingsRequest, requestCreator.getSettings(), GetSettingsResponse.class, headers).next(); - } - - /* - * (non-Javadoc) - * @see org.springframework.data.elasticsearch.client.reactive.ReactiveElasticsearchClient#ping(org.springframework.data.elasticsearch.client.reactive.ReactiveElasticsearchClient.ReactiveElasticsearchClientCallback) - */ - @Override - public Mono execute(ReactiveElasticsearchClientCallback callback) { - - return this.hostProvider - .getActive(HostProvider.Verification.LAZY) // - .flatMap(callback::doWithClient) // - .onErrorResume(throwable -> { - - if (isCausedByConnectionException(throwable)) { - return hostProvider.getActive(HostProvider.Verification.ACTIVE) // - .flatMap(callback::doWithClient); - } - - return Mono.error(throwable); - }); - } - - private boolean isCausedByConnectionException(Throwable throwable) { - - Throwable t = throwable; - do { - - if (t instanceof ConnectException) { - return true; - } - - t = t.getCause(); - } while (t != null); - - return false; - } - - @Override - public Mono status() { - - return hostProvider.clusterInfo() // - .map(it -> new ClientStatus(it.getNodes())); - } - - // --> Private Response helpers - - private static GetResult getResponseToGetResult(GetResponse response) { - - return new GetResult(response.getIndex(), response.getType(), response.getId(), response.getSeqNo(), - response.getPrimaryTerm(), response.getVersion(), response.isExists(), response.getSourceAsBytesRef(), - response.getFields(), null); - } - - // --> - - private Flux sendRequest(REQ request, Function converter, Class responseType, - HttpHeaders headers) { - return sendRequest(converter.apply(request), responseType, headers); - } - - private Flux sendRequest(Request request, Class responseType, HttpHeaders headers) { - - String logId = ClientLogger.newLogId(); - - return Flux - .from(execute(webClient -> sendRequest(webClient, logId, request, headers).exchangeToMono(clientResponse -> { - Publisher publisher = readResponseBody(logId, request, clientResponse, responseType); - return Mono.from(publisher); - }))); - } - - private Flux sendRequest(Req request, - Function converter, - Class responseType, - HttpHeaders headers) { - return sendRequest(request, converter, responseType, headers, DefaultReactiveElasticsearchClient::doDecode); - } - - private Flux sendRequest(Req request, - Function converter, - Class responseType, - HttpHeaders headers, - Function3, String, Mono> decoder) { - return sendRequest(converter.apply(request), responseType, headers, decoder); - } - - private Flux sendRequest(Request request, - Class responseType, - HttpHeaders headers, - Function3, String, Mono> decoder) { - - String logId = ClientLogger.newLogId(); - - return execute(webClient -> Mono.just(this.sendRequest(webClient, logId, request, headers))) - .flatMapMany(spec -> { - return spec.exchangeToFlux(response -> { - return Flux.from( - this.readResponseBody(logId, request, response, responseType, decoder) - ) - .cast(responseType); - }); - }); - } - - private WebClient.RequestBodySpec sendRequest(WebClient webClient, String logId, Request request, HttpHeaders headers) { - - WebClient.RequestBodySpec requestBodySpec = webClient.method(HttpMethod.valueOf(request - .getMethod() - .toUpperCase())) // - .uri(builder -> { - - builder = builder.path(request.getEndpoint()); - - if (!ObjectUtils.isEmpty(request.getParameters())) { - for (Map.Entry entry : request - .getParameters() - .entrySet()) { - builder = builder.queryParam(entry.getKey(), entry.getValue()); - } - } - return builder.build(); - }) // - .attribute(ClientRequest.LOG_ID_ATTRIBUTE, logId) // - .headers(theHeaders -> { - - // add all the headers explicitly set - theHeaders.addAll(headers); - - // and now those that might be set on the request. - if (request.getOptions() != null) { - - if (!ObjectUtils.isEmpty(request - .getOptions() - .getHeaders())) { - request - .getOptions() - .getHeaders() - .forEach(it -> theHeaders.add(it.getName(), it.getValue())); - } - } - }); - - if (request.getEntity() != null) { - - Lazy body = bodyExtractor(request); - - ClientLogger.logRequest(logId, request - .getMethod() - .toUpperCase(), request.getEndpoint(), request.getParameters(), - body::get); - - requestBodySpec.contentType(MediaType.valueOf(request.getEntity().getContentType().getValue())) - .body(Mono.fromSupplier(body), String.class); - } else { - ClientLogger.logRequest(logId, request - .getMethod() - .toUpperCase(), request.getEndpoint(), request.getParameters()); - } - - return requestBodySpec; - } - - private Lazy bodyExtractor(Request request) { - - return Lazy.of(() -> requestBodyToString(request)); - } - - @SneakyThrows - private String requestBodyToString(Request request) { - return EntityUtils.toString(request.getEntity()); - } - - private Publisher readResponseBody(String logId, Request request, ClientResponse response, - Class responseType) { - return readResponseBody(logId, request, response, responseType, DefaultReactiveElasticsearchClient::doDecode); - } - - private Publisher readResponseBody(String logId, - Request request, - ClientResponse response, - Class responseType, - Function3, String, Mono> decoder) { - - if (RawActionResponse.class.equals(responseType)) { - - ClientLogger.logRawResponse(logId, response.statusCode()); - return Mono.just(responseType.cast(RawActionResponse.create(response))); - } - - if (response.statusCode().is5xxServerError()) { - - ClientLogger.logRawResponse(logId, response.statusCode()); - return handleServerError(request, response); - } - - if (response.statusCode().is4xxClientError()) { - - ClientLogger.logRawResponse(logId, response.statusCode()); - return handleClientError(logId, request, response, responseType); - } - - return response.body(BodyExtractors.toMono(byte[].class)) // - .map(it -> new String(it, StandardCharsets.UTF_8)) // - .doOnNext(it -> ClientLogger.logResponse(logId, response.statusCode(), it)) // - .flatMap(content -> decoder.apply(response, responseType, content)); - } - - - private static Mono doDecode(ClientResponse response, Class responseType, String content) { - - String mediaType = response - .headers() - .contentType() - .map(MediaType::toString) - .orElse(XContentType.JSON.mediaType()); - - try { - - Method fromXContent = ReflectionUtils.findMethod(responseType, "fromXContent", XContentParser.class); - if (fromXContent == null) { - fromXContent = ReflectionUtils.findMethod(responseType, "fromXContext", XContentParser.class); - } - return Mono.justOrEmpty(responseType - .cast(ReflectionUtils.invokeMethod(fromXContent, responseType, createParser(mediaType, content)))); - - } catch (Throwable errorParseFailure) { // cause elasticsearch also uses AssertionError - - try { - return Mono.error(BytesRestResponse.errorFromXContent(createParser(mediaType, content))); - } catch (Exception e) { - return Mono - .error(new ElasticsearchStatusException(content, - RestStatus.fromCode(response.statusCode().value()), - errorParseFailure)); - } - } - } - - private static XContentParser createParser(String mediaType, String content) throws IOException { - XContentType type = XContentType.fromMediaTypeOrFormat(mediaType); - if (type == null) { - throw new IOException(content); - } - return XContentType.fromMediaTypeOrFormat(mediaType) // - .xContent() // - .createParser(new NamedXContentRegistry(NamedXContents.getDefaultNamedXContents()), - DeprecationHandler.THROW_UNSUPPORTED_OPERATION, content); - } - - private Publisher handleServerError(Request request, ClientResponse response) { - - int statusCode = response.statusCode().value(); - RestStatus status = RestStatus.fromCode(statusCode); - String mediaType = response - .headers() - .contentType() - .map(MediaType::toString) - .orElse(XContentType.JSON.mediaType()); - - return response - .body(BodyExtractors.toMono(byte[].class)) // - .switchIfEmpty(Mono.error( - () -> new ElasticsearchStatusException(String.format("%s request to %s returned error code %s and no body.", - request.getMethod(), - request.getEndpoint(), - statusCode), - status))) - .map(bytes -> new String(bytes, StandardCharsets.UTF_8)) // - .flatMap(content -> contentOrError(content, mediaType, status)) - .flatMap(unused -> Mono - .error(() -> new ElasticsearchStatusException(String.format("%s request to %s returned error code %s.", - request.getMethod(), - request.getEndpoint(), - statusCode), - status))); - } - - private Publisher handleClientError(String logId, Request request, ClientResponse response, - Class responseType) { - - int statusCode = response.statusCode().value(); - RestStatus status = RestStatus.fromCode(statusCode); - String mediaType = response - .headers() - .contentType() - .map(MediaType::toString) - .orElse(XContentType.JSON.mediaType()); - - return response.body(BodyExtractors.toMono(byte[].class)) // - .map(bytes -> new String(bytes, StandardCharsets.UTF_8)) // - .flatMap(content -> contentOrError(content, mediaType, status)) // - .doOnNext(content -> ClientLogger.logResponse(logId, response.statusCode(), content)) // - .flatMap(content -> doDecode(response, responseType, content)); - } - - // region ElasticsearchException helper - - /** - * checks if the given content body contains an {@link ElasticsearchException}, if yes it is returned in a Mono.error. - * Otherwise the content is returned in the Mono - * - * @param content the content to analyze - * @param mediaType the returned media type - * @param status the response status - * @return a Mono with the content or an Mono.error - */ - private static Mono contentOrError(String content, String mediaType, RestStatus status) { - - ElasticsearchException exception = getElasticsearchException(content, mediaType, status); - - if (exception != null) { - if (status == RestStatus.NOT_FOUND) { - log.warn(exception.getMessage(), exception); - return Mono.empty(); - } - StringBuilder sb = new StringBuilder(); - buildExceptionMessages(sb, exception); - return Mono.error(new ElasticsearchStatusException(sb.toString(), status, exception)); - } - - return Mono.just(content); - } - - /** - * tries to parse an {@link ElasticsearchException} from the given body content - * - * @param content the content to analyse - * @param mediaType the type of the body content - * @return an {@link ElasticsearchException} or {@literal null}. - */ - @Nullable - private static ElasticsearchException getElasticsearchException(String content, String mediaType, RestStatus status) { - - try { - XContentParser parser = createParser(mediaType, content); - // we have a JSON object with an error and a status field - XContentParser.Token token = parser.nextToken(); // Skip START_OBJECT - - do { - token = parser.nextToken(); - - if ("error".equals(parser.currentName())) { - return ElasticsearchException.failureFromXContent(parser); - } - } while (token == XContentParser.Token.FIELD_NAME); - - return null; - } catch (IOException e) { - return new ElasticsearchStatusException(content, status); - } - } - - private static void buildExceptionMessages(StringBuilder sb, Throwable t) { - - sb.append(t.getMessage()); - for (Throwable throwable : t.getSuppressed()) { - sb.append(", "); - buildExceptionMessages(sb, throwable); - } - } - - @Override - public Mono searchForPage(SearchRequest request) { - // if (version.after(Version.V_7_0_0)) { - request.source().trackTotalHits(true); - // } - return this - .sendRequest(request, this::buildSearchRequest, SearchResponse.class, HttpHeaders.EMPTY) - .singleOrEmpty() - .doOnSuccess(res -> log - .trace("execute search {} {} : {}", request.indices(), res.getTook(), request.source())) - .doOnError(err -> log.warn("execute search {} error : {}", request.indices(), request.source(), err)); - } - - @SneakyThrows - protected Request convertMultiSearchRequest(MultiSearchRequest searchRequest) { - Request request = RequestConverters.multiSearch(searchRequest); - if (log.isTraceEnabled()) { - log.trace("execute elasticsearch multi search: {}", requestBodyToString(request)); - } - return request; - } - - @Override - @SneakyThrows - public Mono multiSearch(MultiSearchRequest request) { - Function3, String, Mono> decoder; - if (version.before(Version.V_7_0_0)) { - //适配6.x响应格式 - decoder = (clientResponse, multiSearchResponseClass, s) -> { - JSONObject data = JSON.parseObject(s); - int took = data.getJSONArray("responses") - .stream() - .map(JSONObject.class::cast) - .map(json -> json.getIntValue("took")) - .reduce(Math::addExact) - .orElse(0); - data.put("took", took); - return DefaultReactiveElasticsearchClient.doDecode(clientResponse, multiSearchResponseClass, data.toJSONString()); - }; - } else { - decoder = DefaultReactiveElasticsearchClient::doDecode; - } - return sendRequest(request, - this::convertMultiSearchRequest, - MultiSearchResponse.class, - HttpHeaders.EMPTY, - decoder) - .singleOrEmpty(); - } - - Request convertGetMappingRequest(GetMappingsRequest getMappingsRequest) { - String[] indices = getMappingsRequest.indices() == null ? Strings.EMPTY_ARRAY : getMappingsRequest.indices(); - - Request request = new Request(HttpGet.METHOD_NAME, "/" + String.join(",", indices) + "/_mapping"); - - Params parameters = new Params(request); - parameters.withMasterTimeout(getMappingsRequest.masterNodeTimeout()); - parameters.withIndicesOptions(getMappingsRequest.indicesOptions()); - parameters.withLocal(getMappingsRequest.local()); - parameters.putParam("include_type_name", "true"); - return request; - } - - - @Override - public Mono getMapping(GetMappingsRequest request) { - - return sendRequest(request, this::convertGetMappingRequest, GetMappingsResponse.class, HttpHeaders.EMPTY) - .singleOrEmpty(); - } - - @Override - public Mono getMapping(HttpHeaders headers, GetMappingsRequest getMappingsRequest) { - return sendRequest(getMappingsRequest, requestCreator.getMapping(), - GetMappingsResponse.class, headers).next(); - } - - @Override - public Mono getMapping(HttpHeaders headers, org.elasticsearch.client.indices.GetMappingsRequest getMappingsRequest) { - return sendRequest(getMappingsRequest, requestCreator.getMappingRequest(), org.elasticsearch.client.indices.GetMappingsResponse.class, headers) // - .next(); - } - - @Override - public Mono getFieldMapping(HttpHeaders headers, - GetFieldMappingsRequest getFieldMappingsRequest) { - return sendRequest(getFieldMappingsRequest, requestCreator.getFieldMapping(), GetFieldMappingsResponse.class, - headers).next(); - } - - @Override - public Mono updateAliases(HttpHeaders headers, IndicesAliasesRequest indicesAliasesRequest) { - return sendRequest(indicesAliasesRequest, requestCreator.updateAlias(), AcknowledgedResponse.class, headers) - .map(AcknowledgedResponse::isAcknowledged).next(); - } - - @Override - public Mono getAliases(HttpHeaders headers, GetAliasesRequest getAliasesRequest) { - return sendRequest(getAliasesRequest, requestCreator.getAlias(), GetAliasesResponse.class, headers).next(); - } - - @Override - public Mono putTemplate(HttpHeaders headers, org.elasticsearch.client.indices.PutIndexTemplateRequest putIndexTemplateRequest) { - return sendRequest(putIndexTemplateRequest, requestCreator.putTemplate(), AcknowledgedResponse.class, headers) - .map(AcknowledgedResponse::isAcknowledged).next(); - } - - @Override - public Mono getTemplate(HttpHeaders headers, - org.elasticsearch.client.indices.GetIndexTemplatesRequest getIndexTemplatesRequest) { - return (sendRequest(getIndexTemplatesRequest, requestCreator.getTemplates(), org.elasticsearch.client.indices.GetIndexTemplatesResponse.class, - headers)).next(); - } - - @Override - public Mono existsTemplate(HttpHeaders headers, IndexTemplatesExistRequest indexTemplatesExistRequest) { - return sendRequest(indexTemplatesExistRequest, requestCreator.templatesExist(), - RawActionResponse.class, headers) - .flatMap(response -> response - .releaseBody() - .thenReturn(response.statusCode().is2xxSuccessful())) - .next(); - } - - @Override - public Mono deleteTemplate(HttpHeaders headers, DeleteIndexTemplateRequest deleteIndexTemplateRequest) { - return sendRequest(deleteIndexTemplateRequest, requestCreator.deleteTemplate(), AcknowledgedResponse.class, headers) - .map(AcknowledgedResponse::isAcknowledged).next(); - } - - @Override - public Mono getIndex(HttpHeaders headers, org.elasticsearch.client.indices.GetIndexRequest getIndexRequest) { - return sendRequest(getIndexRequest, requestCreator.getIndex(), GetIndexResponse.class, headers).next(); - } - - Request convertGetIndexTemplateRequest(GetIndexTemplatesRequest getIndexTemplatesRequest) { - Request request = new Request(HttpGet.METHOD_NAME, "/_template/" + String.join(",", getIndexTemplatesRequest.names())); - Params params = new Params(request); - params.putParam("include_type_name", "true"); - return request; - } - - @Override - public Mono getTemplate(GetIndexTemplatesRequest request) { - return sendRequest(request, this::convertGetIndexTemplateRequest, GetIndexTemplatesResponse.class, HttpHeaders.EMPTY) - .singleOrEmpty(); - } - - @SneakyThrows - Request convertPutIndexTemplateRequest(PutIndexTemplateRequest putIndexTemplateRequest) { - Request request = new Request(HttpPut.METHOD_NAME, "/_template/" + putIndexTemplateRequest.name()); - Params params = new Params(request); - params.withMasterTimeout(putIndexTemplateRequest.masterNodeTimeout()); - if (putIndexTemplateRequest.create()) { - params.putParam("create", Boolean.TRUE.toString()); - } - if (Strings.hasText(putIndexTemplateRequest.cause())) { - params.putParam("cause", putIndexTemplateRequest.cause()); - } - params.putParam("include_type_name", "true"); - BytesRef source = XContentHelper.toXContent(putIndexTemplateRequest, XContentType.JSON, false).toBytesRef(); - request.setEntity(new ByteArrayEntity(source.bytes, source.offset, source.length, ContentType.APPLICATION_JSON)); - return request; - } - - @Override - public Mono updateTemplate(PutIndexTemplateRequest request) { - return sendRequest(request, this::convertPutIndexTemplateRequest, AcknowledgedResponse.class, HttpHeaders.EMPTY) - .singleOrEmpty(); - } - - private Version version = Version.CURRENT; - - @Override - public Version serverVersion() { - return version; - } - - @Override - public Mono health(HttpHeaders headers, ClusterHealthRequest clusterHealthRequest) { - return sendRequest(clusterHealthRequest, requestCreator.clusterHealth(), ClusterHealthResponse.class, headers) - .next(); - } - - // endregion - - // region internal classes - - /** - * Reactive client {@link Status} implementation. - * - * @author Christoph Strobl - */ - @Generated - static class ClientStatus implements Status { - - private final Collection connectedHosts; - - ClientStatus(Collection connectedHosts) { - this.connectedHosts = connectedHosts; - } - - /* - * (non-Javadoc) - * @see org.springframework.data.elasticsearch.client.reactive.ReactiveElasticsearchClient.Status#hosts() - */ - @Override - public Collection hosts() { - return connectedHosts; - } - } - - @Generated - static class Params { - private final Request request; - - Params(Request request) { - this.request = request; - } - - Params putParam(String name, String value) { - if (Strings.hasLength(value)) { - request.addParameter(name, value); - } - return this; - } - - Params putParam(String key, TimeValue value) { - if (value != null) { - return putParam(key, value.getStringRep()); - } - return this; - } - - Params withDocAsUpsert(boolean docAsUpsert) { - if (docAsUpsert) { - return putParam("doc_as_upsert", Boolean.TRUE.toString()); - } - return this; - } - - Params withFetchSourceContext(FetchSourceContext fetchSourceContext) { - if (fetchSourceContext != null) { - if (!fetchSourceContext.fetchSource()) { - putParam("_source", Boolean.FALSE.toString()); - } - if (fetchSourceContext.includes() != null && fetchSourceContext.includes().length > 0) { - putParam("_source_includes", String.join(",", fetchSourceContext.includes())); - } - if (fetchSourceContext.excludes() != null && fetchSourceContext.excludes().length > 0) { - putParam("_source_excludes", String.join(",", fetchSourceContext.excludes())); - } - } - return this; - } - - Params withFields(String[] fields) { - if (fields != null && fields.length > 0) { - return putParam("fields", String.join(",", fields)); - } - return this; - } - - Params withMasterTimeout(TimeValue masterTimeout) { - return putParam("master_timeout", masterTimeout); - } - - Params withPipeline(String pipeline) { - return putParam("pipeline", pipeline); - } - - Params withPreference(String preference) { - return putParam("preference", preference); - } - - Params withRealtime(boolean realtime) { - if (!realtime) { - return putParam("realtime", Boolean.FALSE.toString()); - } - return this; - } - - Params withRefresh(boolean refresh) { - if (refresh) { - return withRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - } - return this; - } - - Params withRefreshPolicy(WriteRequest.RefreshPolicy refreshPolicy) { - if (refreshPolicy != WriteRequest.RefreshPolicy.NONE) { - return putParam("refresh", refreshPolicy.getValue()); - } - return this; - } - - Params withRequestsPerSecond(float requestsPerSecond) { - // the default in AbstractBulkByScrollRequest is Float.POSITIVE_INFINITY, - // but we don't want to add that to the URL parameters, instead we use -1 - if (Float.isFinite(requestsPerSecond)) { - return putParam("requests_per_second", Float.toString(requestsPerSecond)); - } else { - return putParam("requests_per_second", "-1"); - } - } - - Params withRetryOnConflict(int retryOnConflict) { - if (retryOnConflict > 0) { - return putParam("retry_on_conflict", String.valueOf(retryOnConflict)); - } - return this; - } - - Params withRouting(String routing) { - return putParam("routing", routing); - } - - Params withStoredFields(String[] storedFields) { - if (storedFields != null && storedFields.length > 0) { - return putParam("stored_fields", String.join(",", storedFields)); - } - return this; - } - - Params withTimeout(TimeValue timeout) { - return putParam("timeout", timeout); - } - - Params withVersion(long version) { - if (version != Versions.MATCH_ANY) { - return putParam("version", Long.toString(version)); - } - return this; - } - - Params withVersionType(VersionType versionType) { - if (versionType != VersionType.INTERNAL) { - return putParam("version_type", versionType.name().toLowerCase(Locale.ROOT)); - } - return this; - } - - Params withIfSeqNo(long seqNo) { - if (seqNo != SequenceNumbers.UNASSIGNED_SEQ_NO) { - return putParam("if_seq_no", Long.toString(seqNo)); - } - return this; - } - - Params withIfPrimaryTerm(long primaryTerm) { - if (primaryTerm != SequenceNumbers.UNASSIGNED_PRIMARY_TERM) { - return putParam("if_primary_term", Long.toString(primaryTerm)); - } - return this; - } - - Params withWaitForActiveShards(ActiveShardCount activeShardCount) { - return withWaitForActiveShards(activeShardCount, ActiveShardCount.DEFAULT); - } - - Params withWaitForActiveShards(ActiveShardCount activeShardCount, ActiveShardCount defaultActiveShardCount) { - if (activeShardCount != null && activeShardCount != defaultActiveShardCount) { - // in Elasticsearch 7, "default" cannot be sent anymore, so it needs to be mapped to the default value of 1 - String value = activeShardCount == ActiveShardCount.DEFAULT ? "1" - : activeShardCount.toString().toLowerCase(Locale.ROOT); - return putParam("wait_for_active_shards", value); - } - return this; - } - - Params withIndicesOptions(IndicesOptions indicesOptions) { - withIgnoreUnavailable(indicesOptions.ignoreUnavailable()); - putParam("allow_no_indices", Boolean.toString(indicesOptions.allowNoIndices())); - String expandWildcards; - if (!indicesOptions.expandWildcardsOpen() && !indicesOptions.expandWildcardsClosed()) { - expandWildcards = "none"; - } else { - StringJoiner joiner = new StringJoiner(","); - if (indicesOptions.expandWildcardsOpen()) { - joiner.add("open"); - } - if (indicesOptions.expandWildcardsClosed()) { - joiner.add("closed"); - } - expandWildcards = joiner.toString(); - } - putParam("expand_wildcards", expandWildcards); - return this; - } - - Params withIgnoreUnavailable(boolean ignoreUnavailable) { - // Always explicitly place the ignore_unavailable value. - putParam("ignore_unavailable", Boolean.toString(ignoreUnavailable)); - return this; - } - - Params withHuman(boolean human) { - if (human) { - putParam("human", "true"); - } - return this; - } - - Params withLocal(boolean local) { - if (local) { - putParam("local", "true"); - } - return this; - } - - Params withIncludeDefaults(boolean includeDefaults) { - if (includeDefaults) { - return putParam("include_defaults", Boolean.TRUE.toString()); - } - return this; - } - - Params withPreserveExisting(boolean preserveExisting) { - if (preserveExisting) { - return putParam("preserve_existing", Boolean.TRUE.toString()); - } - return this; - } - - Params withDetailed(boolean detailed) { - if (detailed) { - return putParam("detailed", Boolean.TRUE.toString()); - } - return this; - } - - Params withWaitForCompletion(Boolean waitForCompletion) { - return putParam("wait_for_completion", waitForCompletion.toString()); - } - - Params withNodes(String[] nodes) { - if (nodes != null && nodes.length > 0) { - return putParam("nodes", String.join(",", nodes)); - } - return this; - } - - Params withActions(String[] actions) { - if (actions != null && actions.length > 0) { - return putParam("actions", String.join(",", actions)); - } - return this; - } - - Params withTaskId(TaskId taskId) { - if (taskId != null && taskId.isSet()) { - return putParam("task_id", taskId.toString()); - } - return this; - } - - Params withParentTaskId(TaskId parentTaskId) { - if (parentTaskId != null && parentTaskId.isSet()) { - return putParam("parent_task_id", parentTaskId.toString()); - } - return this; - } - - Params withVerify(boolean verify) { - if (verify) { - return putParam("verify", Boolean.TRUE.toString()); - } - return this; - } - - Params withWaitForStatus(ClusterHealthStatus status) { - if (status != null) { - return putParam("wait_for_status", status.name().toLowerCase(Locale.ROOT)); - } - return this; - } - - Params withWaitForNoRelocatingShards(boolean waitNoRelocatingShards) { - if (waitNoRelocatingShards) { - return putParam("wait_for_no_relocating_shards", Boolean.TRUE.toString()); - } - return this; - } - - - Params withWaitForNoInitializingShards(boolean waitNoInitShards) { - if (waitNoInitShards) { - return putParam("wait_for_no_initializing_shards", Boolean.TRUE.toString()); - } - return this; - } - - Params withWaitForNodes(String waitForNodes) { - return putParam("wait_for_nodes", waitForNodes); - } - - Params withLevel(ClusterHealthRequest.Level level) { - return putParam("level", level.name().toLowerCase(Locale.ROOT)); - } - - Params withWaitForEvents(Priority waitForEvents) { - if (waitForEvents != null) { - return putParam("wait_for_events", waitForEvents.name().toLowerCase(Locale.ROOT)); - } - return this; - } - - } - -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/reactive/RawActionResponse.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/reactive/RawActionResponse.java deleted file mode 100755 index 5e803989..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/reactive/RawActionResponse.java +++ /dev/null @@ -1,70 +0,0 @@ -package org.jetlinks.community.elastic.search.service.reactive; - -import lombok.Generated; -import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.springframework.http.HttpStatus; -import org.springframework.http.client.reactive.ClientHttpResponse; -import org.springframework.web.reactive.function.BodyExtractor; -import org.springframework.web.reactive.function.client.ClientResponse; -import reactor.core.publisher.Mono; - -import java.io.IOException; - -/** - * Extension to {@link ActionResponse} that also delegates to {@link ClientResponse}. - * - * @author Christoph Strobl - * @author Peter-Josef Meisch - * @author Mark Paluch - * @since 3.2 - */ -@Generated -class RawActionResponse extends ActionResponse { - - private final ClientResponse delegate; - - private RawActionResponse(ClientResponse delegate) { - this.delegate = delegate; - } - - static RawActionResponse create(ClientResponse response) { - return new RawActionResponse(response); - } - - public HttpStatus statusCode() { - return delegate.statusCode(); - } - - /* - * (non-Javadoc) - * @see org.springframework.web.reactive.function.client.ClientResponse#headers() - */ - public ClientResponse.Headers headers() { - return delegate.headers(); - } - - /* - * (non-Javadoc) - * @see org.springframework.web.reactive.function.client.ClientResponse#body(org.springframework.web.reactive.function.BodyExtractor) - */ - public T body(BodyExtractor extractor) { - return delegate.body(extractor); - } - - /* - * (non-Javadoc) - * until Elasticsearch 7.4 this empty implementation was available in the abstract base class - */ - @Override - public void writeTo(StreamOutput out) throws IOException { - } - - /** - * Ensure the response body is released to properly release the underlying connection. - * - */ - public Mono releaseBody() { - return delegate.releaseBody(); - } -} \ No newline at end of file diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/reactive/ReactiveAggregationService.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/reactive/ReactiveAggregationService.java deleted file mode 100755 index 50040de8..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/reactive/ReactiveAggregationService.java +++ /dev/null @@ -1,356 +0,0 @@ -package org.jetlinks.community.elastic.search.service.reactive; - -import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.Version; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.search.aggregations.*; -import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; -import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; -import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; -import org.elasticsearch.search.aggregations.bucket.histogram.LongBounds; -import org.elasticsearch.search.aggregations.bucket.terms.Terms; -import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.*; -import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.sort.SortBuilders; -import org.elasticsearch.search.sort.SortOrder; -import org.hswebframework.ezorm.core.param.QueryParam; -import org.hswebframework.ezorm.core.param.Term; -import org.hswebframework.ezorm.core.param.TermType; -import org.jetlinks.core.metadata.types.DateTimeType; -import org.jetlinks.community.Interval; -import org.jetlinks.community.elastic.search.index.ElasticSearchIndexManager; -import org.jetlinks.community.elastic.search.service.AggregationService; -import org.jetlinks.community.elastic.search.utils.ElasticSearchConverter; -import org.jetlinks.community.timeseries.query.*; -import org.jetlinks.reactor.ql.utils.CastUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.util.CollectionUtils; -import org.springframework.util.StringUtils; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; - -import java.time.Duration; -import java.time.ZoneId; -import java.util.*; -import java.util.stream.Collectors; - -/** - * @author zhouhao - * @since 1.5 - **/ -@Slf4j -public class ReactiveAggregationService implements AggregationService { - - private final ReactiveElasticsearchClient restClient; - - private final ElasticSearchIndexManager indexManager; - - //是否打开执行提示，打开后，聚合查询时会给每一个bucket构造global ordinals。单个索引数据量大于一百万时建议关闭。 - private static final boolean IS_OPEN_GLOBAL_ORDINALS = - Boolean.parseBoolean(System.getProperty("elasticsearch.agg.query.execution_hint", "false")); - - @Autowired - public ReactiveAggregationService(ElasticSearchIndexManager indexManager, - ReactiveElasticsearchClient restClient) { - this.restClient = restClient; - this.indexManager = indexManager; - } - - private Mono createSearchSourceBuilder(QueryParam queryParam, String index) { - - return indexManager - .getIndexMetadata(index) - .map(metadata -> ElasticSearchConverter.convertSearchSourceBuilder(queryParam, metadata)); - } - - private AggregationBuilder createBuilder(Group group, AggregationQueryParam param) { - - if (group instanceof TimeGroup) { - TimeGroup timeGroup = ((TimeGroup) group); - DateHistogramAggregationBuilder builder = AggregationBuilders - .dateHistogram(timeGroup.getAlias()) - .field(timeGroup.getProperty()); - if (StringUtils.hasText(timeGroup.getFormat())) { - String format = timeGroup.getFormat(); - if (format.startsWith("yyyy")) { - format = "8" + format; - } - builder.format(format); - } - builder.timeZone(ZoneId.systemDefault()); - builder.order(BucketOrder.key(false)); - if (timeGroup.getInterval() != null) { - Interval interval = timeGroup.getInterval(); - String intervalString = interval.toString(); - if (restClient.serverVersion().after(Version.V_7_2_0)) { - if (DateHistogramAggregationBuilder.DATE_FIELD_UNITS.containsKey(intervalString)) { - builder.calendarInterval(new DateHistogramInterval(intervalString)); - } else { - builder.fixedInterval(new DateHistogramInterval(intervalString)); -// builder.dateHistogramInterval(new DateHistogramInterval(intervalString)); - } - } else { - builder.dateHistogramInterval(new DateHistogramInterval(intervalString)); - } - } - - builder.extendedBounds(getExtendedBounds(param)); -// builder.missing(""); - - return builder; - } else { - TermsAggregationBuilder builder = AggregationBuilders - .terms(group.getAlias()) - .field(group.getProperty()); - if (group instanceof LimitGroup) { - if (((LimitGroup) group).getLimit() > 0) { - builder.size(((LimitGroup) group).getLimit()); - } - } else { - builder.size(100); - } - //直接进行子聚合的计算 - builder.collectMode(Aggregator.SubAggCollectionMode.DEPTH_FIRST); -// builder.missing(0); - if (IS_OPEN_GLOBAL_ORDINALS) { - builder.executionHint("map"); - } - return builder; - } - } - - @Override - public Flux> aggregation(String[] index, AggregationQueryParam aggregationQueryParam) { - QueryParam queryParam = prepareQueryParam(aggregationQueryParam); - - List groups = new ArrayList<>(); - // TODO: 2020/9/3 - if (aggregationQueryParam.getGroupByTime() != null) { - groups.add(aggregationQueryParam.getGroupByTime()); - } - groups.addAll(aggregationQueryParam.getGroupBy()); - List aggs = new ArrayList<>(); - - AggregationBuilder aggregationBuilder = null; - AggregationBuilder lastAgg = null; - if (!groups.isEmpty()) { - Group first = groups.get(0); - aggregationBuilder = lastAgg = createBuilder(first, aggregationQueryParam); - for (int i = 1; i < groups.size(); i++) { - aggregationBuilder.subAggregation(lastAgg = createBuilder(groups.get(i), aggregationQueryParam)); - } - aggs.add(aggregationBuilder); - } - - boolean group = aggregationBuilder != null; - for (AggregationColumn aggColumn : aggregationQueryParam.getAggColumns()) { - AggregationBuilder builder = AggType - .of(aggColumn.getAggregation().name()) - .aggregationBuilder(aggColumn.getAlias(), aggColumn.getProperty()); - - if (builder instanceof ValuesSourceAggregationBuilder && - aggColumn.getDefaultValue() != null) { - ((ValuesSourceAggregationBuilder) builder) - .missing(aggColumn.getDefaultValue()); - } - - if (builder instanceof TopHitsAggregationBuilder) { - TopHitsAggregationBuilder topHitsBuilder = ((TopHitsAggregationBuilder) builder); - if (CollectionUtils.isEmpty(queryParam.getSorts())) { - topHitsBuilder.sort(aggregationQueryParam.getTimeProperty(), SortOrder.DESC); - } else { - topHitsBuilder.sorts(queryParam - .getSorts() - .stream() - .map(sort -> SortBuilders - .fieldSort(sort.getName()) - .order("desc".equalsIgnoreCase(sort.getOrder()) - ? SortOrder.DESC - : SortOrder.ASC)) - .collect(Collectors.toList())); - } - if (aggColumn instanceof LimitAggregationColumn) { - topHitsBuilder.size(((LimitAggregationColumn) aggColumn).getLimit()); - } else { - topHitsBuilder.size(1); - } - } - if (group) { - lastAgg.subAggregation(builder); - } else { - aggs.add(builder); - } - } - - return Flux - .fromArray(index) - .flatMap(idx -> Mono.zip(indexManager.getIndexStrategy(idx), Mono.just(idx))) - .collectList() - .flatMap(strategy -> this - .createSearchSourceBuilder(queryParam, index[0]) - .map(builder -> { - aggs.forEach(builder.size(0)::aggregation); - return new SearchRequest(strategy - .stream() - .map(tp2 -> tp2 - .getT1() - .getIndexForSearch(tp2.getT2())) - .toArray(String[]::new)) - .indicesOptions(ReactiveElasticSearchService.indexOptions) - .source(builder); - } - ) - ) - .flatMap(restClient::searchForPage) - .flatMapMany(this::parseResult) - .as(flux -> { - if (!group) { - return flux - .map(Map::entrySet) - .flatMap(Flux::fromIterable) - .collectMap(Map.Entry::getKey, Map.Entry::getValue) - .flux(); - } - return flux; - }) - ; - } - - protected Flux> parseResult(SearchResponse searchResponse) { - return Mono.justOrEmpty(searchResponse.getAggregations()) - .flatMapIterable(Aggregations::asList) - .flatMap(agg -> parseAggregation(agg.getName(), agg), Integer.MAX_VALUE); - } - - private Flux> parseAggregation(String name, - org.elasticsearch.search.aggregations.Aggregation aggregation) { - if (aggregation instanceof Terms) { - return parseAggregation(((Terms) aggregation)); - } - if (aggregation instanceof TopHits) { - TopHits topHits = ((TopHits) aggregation); - return Flux - .fromArray(topHits.getHits().getHits()) - .map(hit -> { - Map val = hit.getSourceAsMap(); - if (!val.containsKey("id")) { - val.put("id", hit.getId()); - } - return val; - }); - } - if (aggregation instanceof Histogram) { - return parseAggregation(((Histogram) aggregation)); - } - if (aggregation instanceof ValueCount) { - return Flux.just(Collections.singletonMap(name, ((ValueCount) aggregation).getValue())); - } - if (aggregation instanceof NumericMetricsAggregation.SingleValue) { - return Flux.just(Collections.singletonMap(name, getSafeNumber(((NumericMetricsAggregation.SingleValue) aggregation) - .value()))); - } - if (aggregation instanceof ExtendedStats) { - ExtendedStats stats = ((ExtendedStats) aggregation); - // TODO: 2020/10/29 只处理了标准差差 - return Flux.just(Collections.singletonMap(name, stats.getStdDeviation())); - } - - return Flux.empty(); - } - - private Object getSafeNumber(double number) { - return (Double.isNaN(number) || Double.isInfinite(number)) ? null : number; - } - - private Flux> parseAggregation(Histogram aggregation) { - - return Flux - .fromIterable(aggregation.getBuckets()) - .flatMap(bucket -> - Flux.fromIterable(bucket.getAggregations().asList()) - .flatMap(agg -> this.parseAggregation(agg.getName(), agg), Integer.MAX_VALUE) - .defaultIfEmpty(Collections.emptyMap()) -// .map(Map::entrySet) -// .flatMap(Flux::fromIterable) -// .collectMap(Map.Entry::getKey, Map.Entry::getValue) - .map(map -> { - Map val = new HashMap<>(map); - val.put(aggregation.getName(), bucket.getKeyAsString()); - val.put("_" + aggregation.getName(), bucket.getKey()); - return val; - }), - Integer.MAX_VALUE - ); - } - - private Flux> parseAggregation(Terms aggregation) { - - return Flux.fromIterable(aggregation.getBuckets()) - .flatMap(bucket -> Flux.fromIterable(bucket.getAggregations().asList()) - .flatMap(agg -> parseAggregation(agg.getName(), agg) - .map(map -> { - Map val = new HashMap<>(map); - val.put(aggregation.getName(), bucket.getKeyAsString()); - return val; - }) - )); - } - - protected static QueryParam prepareQueryParam(AggregationQueryParam param) { - QueryParam queryParam = param.getQueryParam().clone(); - queryParam.setPaging(false); - boolean hasTimestamp = false; - for (Term term : queryParam.getTerms()) { - if (param.getTimeProperty().equals(term.getColumn())) { - hasTimestamp = true; - } - } - if (!hasTimestamp) { - queryParam.and(param.getTimeProperty(), TermType.btw, Arrays.asList(calculateStartWithTime(param), param.getEndWithTime())); - } - if (queryParam.getSorts().isEmpty()) { - queryParam.orderBy(param.getTimeProperty()).desc(); - } - return queryParam; - } - - protected static LongBounds getExtendedBounds(AggregationQueryParam param) { - - return new LongBounds(calculateStartWithTime(param), param.getEndWithTime()); - } - - //聚合查询默认的时间间隔 - static long thirtyDayMillis = Duration - .ofDays(Integer.getInteger("elasticsearch.agg.default-range-day", 90)) - .toMillis(); - - static long calculateStartWithTime(AggregationQueryParam param) { - long startWithParam = param.getStartWithTime(); - if (startWithParam == 0) { - //从查询条件中提取时间参数来获取时间区间 - List terms = param.getQueryParam().getTerms(); - for (Term term : terms) { - if ("timestamp".equals(term.getColumn())) { - Object value = term.getValue(); - String termType = term.getTermType(); - if (TermType.btw.equals(termType)) { - if (String.valueOf(value).contains(",")) { - value = Arrays.asList(String.valueOf(value).split(",")); - } - return DateTimeType.GLOBAL.convert(CastUtils.castArray(value).get(0)).getTime(); - } - if (TermType.gt.equals(termType) || TermType.gte.equals(termType)) { - - return DateTimeType.GLOBAL.convert(value).getTime(); - } - } - } - return param.getEndWithTime() - thirtyDayMillis; - } - return startWithParam; - } - -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/reactive/ReactiveElasticsearchClient.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/reactive/ReactiveElasticsearchClient.java deleted file mode 100755 index 6c3d7730..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/reactive/ReactiveElasticsearchClient.java +++ /dev/null @@ -1,31 +0,0 @@ -package org.jetlinks.community.elastic.search.service.reactive; - -import org.elasticsearch.Version; -import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; -import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; -import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesRequest; -import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse; -import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; -import org.elasticsearch.action.search.MultiSearchRequest; -import org.elasticsearch.action.search.MultiSearchResponse; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.support.master.AcknowledgedResponse; -import reactor.core.publisher.Mono; - -public interface ReactiveElasticsearchClient extends - org.springframework.data.elasticsearch.client.reactive.ReactiveElasticsearchClient - , org.springframework.data.elasticsearch.client.reactive.ReactiveElasticsearchClient.Indices { - - Mono searchForPage(SearchRequest request); - - Mono multiSearch(MultiSearchRequest request); - - Mono getMapping(GetMappingsRequest request); - - Mono getTemplate(GetIndexTemplatesRequest request); - - Mono updateTemplate(PutIndexTemplateRequest request); - - Version serverVersion(); -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchRowModeSaveOperations.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchRowModeSaveOperations.java deleted file mode 100644 index e2ff31ec..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/things/ElasticSearchRowModeSaveOperations.java +++ /dev/null @@ -1,33 +0,0 @@ -package org.jetlinks.community.elastic.search.things; - -import org.jetlinks.core.things.ThingsRegistry; -import org.jetlinks.community.elastic.search.service.ElasticSearchService; -import org.jetlinks.community.things.data.operations.DataSettings; -import org.jetlinks.community.things.data.operations.MetricBuilder; -import org.jetlinks.community.things.data.operations.RowModeSaveOperationsBase; -import org.jetlinks.community.timeseries.TimeSeriesData; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; - -class ElasticSearchRowModeSaveOperations extends RowModeSaveOperationsBase { - - private final ElasticSearchService searchService; - - public ElasticSearchRowModeSaveOperations(ThingsRegistry registry, - MetricBuilder metricBuilder, - DataSettings settings, - ElasticSearchService searchService) { - super(registry, metricBuilder, settings); - this.searchService = searchService; - } - - @Override - protected Mono doSave(String metric, TimeSeriesData data) { - return searchService.commit(metric, data.getData()); - } - - @Override - protected Mono doSave(String metric, Flux data) { - return searchService.save(metric, data.map(TimeSeriesData::getData)); - } -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/utils/QueryParamTranslator.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/utils/QueryParamTranslator.java deleted file mode 100644 index c24a1d42..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/utils/QueryParamTranslator.java +++ /dev/null @@ -1,125 +0,0 @@ -package org.jetlinks.community.elastic.search.utils; - -import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.sort.SortOrder; -import org.hswebframework.ezorm.core.param.QueryParam; -import org.hswebframework.ezorm.core.param.Sort; -import org.hswebframework.ezorm.core.param.Term; -import org.hswebframework.ezorm.core.param.TermType; -import org.jetlinks.community.elastic.search.index.ElasticSearchIndexMetadata; -import org.jetlinks.community.elastic.search.parser.DefaultLinkTypeParser; -import org.jetlinks.community.utils.ConverterUtils; -import org.jetlinks.community.utils.TimeUtils; -import org.jetlinks.core.metadata.Converter; -import org.jetlinks.core.metadata.DataType; -import org.jetlinks.core.metadata.PropertyMetadata; -import org.jetlinks.core.metadata.types.DateTimeType; -import org.springframework.util.ObjectUtils; -import org.springframework.util.StringUtils; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.function.BiConsumer; -import java.util.function.Consumer; - -/** - * @author zhouhao - * @since 1.0 - **/ -@Slf4j -public class QueryParamTranslator { - - static DefaultLinkTypeParser linkTypeParser = new DefaultLinkTypeParser(); - - static Consumer doNotingParamConverter = (term -> { - }); - - static Map> converter = new ConcurrentHashMap<>(); - - static BiConsumer defaultDataTypeConverter = (type, term) -> { - - }; - - private static boolean maybeList(Term term) { - switch (term.getTermType().toLowerCase()) { - case TermType.in: - case TermType.nin: - case TermType.btw: - case TermType.nbtw: - return true; - } - return false; - } - - private static boolean isDoNotConvertValue(Term term) { - switch (term.getTermType().toLowerCase()) { - case TermType.isnull: - case TermType.notnull: - case TermType.empty: - case TermType.nempty: - return true; - } - return false; - } - - - private static Object convertValue(DataType type, Object val) { - if (type instanceof DateTimeType) { - return TimeUtils.convertToDate(val).getTime(); - } else if (type instanceof Converter) { - return ((Converter) type).convert(val); - } - return val; - } - - public static QueryBuilder createQueryBuilder(QueryParam queryParam, ElasticSearchIndexMetadata metadata) { - BoolQueryBuilder queryBuilders = QueryBuilders.boolQuery(); - Consumer paramConverter = doNotingParamConverter; - if (metadata != null) { - paramConverter = t -> { - if (ObjectUtils.isEmpty(t.getColumn()) || isDoNotConvertValue(t)) { - return; - } - PropertyMetadata property = metadata.getProperty(t.getColumn()); - if (null != property) { - DataType type = property.getValueType(); - - Object value; - if (maybeList(t)) { - value = ConverterUtils.tryConvertToList(t.getValue(), v -> convertValue(type, v)); - } else { - value = convertValue(type, t.getValue()); - } - if (null != value) { - t.setValue(value); - } else { - log.warn("Can not convert {} to {}", t.getValue(), type.getId()); - } - converter.getOrDefault(type.getId(), defaultDataTypeConverter).accept(type, t); - } - }; - } - linkTypeParser.process(queryParam.getTerms(), paramConverter, queryBuilders); - return queryBuilders; - } - - public static SearchSourceBuilder convertSearchSourceBuilder(QueryParam queryParam, ElasticSearchIndexMetadata metadata) { - SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); - if (queryParam.isPaging()) { - sourceBuilder.from(queryParam.getPageIndex() * queryParam.getPageSize()); - sourceBuilder.size(queryParam.getPageSize()); - } - for (Sort sort : queryParam.getSorts()) { - if (!StringUtils.isEmpty(sort.getName())) { - sourceBuilder.sort(sort.getName(), SortOrder.fromString(sort.getOrder())); - } - } - - return sourceBuilder.query(createQueryBuilder(queryParam, metadata)); - } - -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/utils/ReactorActionListener.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/utils/ReactorActionListener.java deleted file mode 100644 index d1f60afb..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/utils/ReactorActionListener.java +++ /dev/null @@ -1,62 +0,0 @@ -package org.jetlinks.community.elastic.search.utils; - -import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.ElasticsearchStatusException; -import org.elasticsearch.action.ActionListener; -import org.hswebframework.web.exception.BusinessException; -import reactor.core.publisher.Mono; - -import java.util.function.Consumer; -import java.util.function.Function; - -@Slf4j -public class ReactorActionListener { - - public static Mono mono(Consumer> listenerConsumer, - Function> onSuccess, - Function> onError) { - return Mono.>create(sink -> { - listenerConsumer.accept(new ActionListener() { - @Override - public void onResponse(T t) { - try { - sink.success(onSuccess.apply(t)); - } catch (Exception e) { - sink.error(e); - } - } - - @Override - public void onFailure(Exception e) { - try { - sink.success(onError.apply(e)); - } catch (Exception e2) { - sink.error(e2); - } - } - }); - - }).flatMap(Function.identity()) - .onErrorResume(ElasticsearchStatusException.class, e -> { - if (e.status().getStatus() == 404) { - if(e.getMessage().contains("index_not_found_exception")){ - log.debug("{}",e.getMessage()); - }else{ - log.warn("{}",e.getDetailedMessage(),e); - } - return Mono.empty(); - } - return Mono.error(new BusinessException(e.getMessage(), e)); - }); - } - - - public static Mono mono(Consumer> listenerConsumer, - Function> onSuccess) { - return mono(listenerConsumer, onSuccess, Mono::error); - } - - public static Mono mono(Consumer> listenerConsumer) { - return mono(listenerConsumer, Mono::justOrEmpty, Mono::error); - } -} diff --git a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/utils/TermCommonUtils.java b/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/utils/TermCommonUtils.java deleted file mode 100644 index 767670b8..00000000 --- a/jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/utils/TermCommonUtils.java +++ /dev/null @@ -1,36 +0,0 @@ -package org.jetlinks.community.elastic.search.utils; - -import java.util.*; - -/** - * @author bsetfeng - * @since 1.0 - **/ -public class TermCommonUtils { - - public static List convertToList(Object value) { - if (value == null) { - return Collections.emptyList(); - } - if (value instanceof String) { - value = ((String) value).split(","); - } - - if (value instanceof Object[]) { - value = Arrays.asList(((Object[]) value)); - } - - if (value instanceof Collection) { - return new ArrayList(((Collection) value)); - } - - return Collections.singletonList(value); - } - - public static Object getStandardsTermValue(List value) { - if (value.size() == 1) { - return value.get(0); - } - return value; - } -} diff --git a/jetlinks-components/gateway-component/pom.xml b/jetlinks-components/gateway-component/pom.xml index 1e8100b2..b0739014 100644 --- a/jetlinks-components/gateway-component/pom.xml +++ b/jetlinks-components/gateway-component/pom.xml @@ -5,7 +5,7 @@ jetlinks-components org.jetlinks.community - 2.3.0-SNAPSHOT + 2.10.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/jetlinks-components/gateway-component/src/main/java/org/jetlinks/community/gateway/DeviceGatewayHelper.java b/jetlinks-components/gateway-component/src/main/java/org/jetlinks/community/gateway/DeviceGatewayHelper.java index 293331bb..6fe0cb85 100755 --- a/jetlinks-components/gateway-component/src/main/java/org/jetlinks/community/gateway/DeviceGatewayHelper.java +++ b/jetlinks-components/gateway-component/src/main/java/org/jetlinks/community/gateway/DeviceGatewayHelper.java @@ -20,7 +20,7 @@ import org.springframework.util.StringUtils; import reactor.core.publisher.Mono; import reactor.util.context.Context; -import javax.validation.constraints.NotNull; +import jakarta.validation.constraints.NotNull; import java.time.Duration; import java.util.Objects; import java.util.function.Consumer; diff --git a/jetlinks-components/gateway-component/src/main/java/org/jetlinks/community/gateway/external/socket/WebSocketMessagingHandler.java b/jetlinks-components/gateway-component/src/main/java/org/jetlinks/community/gateway/external/socket/WebSocketMessagingHandler.java index 8b1bee20..38e3c386 100644 --- a/jetlinks-components/gateway-component/src/main/java/org/jetlinks/community/gateway/external/socket/WebSocketMessagingHandler.java +++ b/jetlinks-components/gateway-component/src/main/java/org/jetlinks/community/gateway/external/socket/WebSocketMessagingHandler.java @@ -40,6 +40,7 @@ public class WebSocketMessagingHandler implements WebSocketHandler { // /messaging/{token} @Override @Nonnull + @SuppressWarnings("all") public Mono handle(@Nonnull WebSocketSession session) { String[] path = session.getHandshakeInfo().getUri().getPath().split("[/]"); if (path.length == 0) { @@ -118,8 +119,7 @@ public class WebSocketMessagingHandler implements WebSocketHandler { subs.remove(request.getId()); }) .transform(session::send) - .subscriberContext(ReactiveLogger.start(context)) - .subscriberContext(Context.of(Authentication.class, auth)) + .contextWrite(Context.of(Authentication.class, auth)) .subscribe(); if (!sub.isDisposed()) { subs.put(request.getId(), sub); diff --git a/jetlinks-components/gateway-component/src/main/java/org/jetlinks/community/gateway/spring/EventBusDispatcherConfiguration.java b/jetlinks-components/gateway-component/src/main/java/org/jetlinks/community/gateway/spring/EventBusDispatcherConfiguration.java new file mode 100644 index 00000000..96739cd5 --- /dev/null +++ b/jetlinks-components/gateway-component/src/main/java/org/jetlinks/community/gateway/spring/EventBusDispatcherConfiguration.java @@ -0,0 +1,17 @@ +package org.jetlinks.community.gateway.spring; + +import org.springframework.beans.factory.config.BeanDefinition; +import org.springframework.boot.autoconfigure.AutoConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Role; + +@AutoConfiguration +@Role(BeanDefinition.ROLE_INFRASTRUCTURE) +public class EventBusDispatcherConfiguration { + + @Bean + @Role(BeanDefinition.ROLE_INFRASTRUCTURE) + public SpringMessageBroker springMessageBroker() { + return new SpringMessageBroker(); + } +} diff --git a/jetlinks-components/gateway-component/src/main/java/org/jetlinks/community/gateway/spring/ProxyMessageListener.java b/jetlinks-components/gateway-component/src/main/java/org/jetlinks/community/gateway/spring/ProxyMessageListener.java index 2f24b420..174ad851 100644 --- a/jetlinks-components/gateway-component/src/main/java/org/jetlinks/community/gateway/spring/ProxyMessageListener.java +++ b/jetlinks-components/gateway-component/src/main/java/org/jetlinks/community/gateway/spring/ProxyMessageListener.java @@ -1,12 +1,9 @@ package org.jetlinks.community.gateway.spring; +import io.netty.util.ReferenceCountUtil; import lombok.extern.slf4j.Slf4j; import org.hswebframework.web.bean.FastBeanCopier; import org.hswebframework.web.proxy.Proxy; -import org.jetlinks.core.NativePayload; -import org.jetlinks.core.Payload; -import org.jetlinks.core.codec.Codecs; -import org.jetlinks.core.codec.Decoder; import org.jetlinks.core.event.TopicPayload; import org.reactivestreams.Publisher; import org.springframework.core.ResolvableType; @@ -26,7 +23,10 @@ class ProxyMessageListener implements MessageListener { private final Method method; private final BiFunction proxy; - private volatile Decoder decoder; + private final boolean paramIsPayload; + + private final boolean paramIsVoid; + @SuppressWarnings("all") ProxyMessageListener(Object target, Method method) { @@ -51,6 +51,9 @@ class ProxyMessageListener implements MessageListener { String invokeCode; if (paramType != Void.class) { + if (paramType.isPrimitive()) { + throw new UnsupportedOperationException(method + "参数不支持基本数据类型,请使用包装器类型."); + } code.add(paramType.getName() + " _param = (" + paramType.getName() + ")param;"); invokeCode = " _target." + method.getName() + "(_param);"; } else { @@ -64,57 +67,45 @@ class ProxyMessageListener implements MessageListener { } code.add("}"); - this.resolvableType = ResolvableType.forMethodParameter(method, 0, targetType); + if (paramType == Void.class) { + this.resolvableType = ResolvableType.forClass(Void.class); + } else { + this.resolvableType = ResolvableType.forMethodParameter(method, 0, targetType); + } this.proxy = Proxy.create(BiFunction.class) - .addMethod(code.toString()) - .newInstance(); + .addMethod(code.toString()) + .newInstance(); + paramIsPayload = TopicPayload.class.isAssignableFrom(paramType); + paramIsVoid = paramType == Void.class; } Object convert(TopicPayload message) { - - if (Payload.class.isAssignableFrom(paramType)) { + if (paramIsPayload) { return message; } try { - Payload payload = message.getPayload(); - Object decodedPayload; - if (payload instanceof NativePayload) { - decodedPayload = ((NativePayload) payload).getNativeObject(); - } else { - if (decoder == null) { - decoder = Codecs.lookup(resolvableType); - } - decodedPayload = decoder.decode(message); - } + Object decodedPayload = message.decode(false); if (paramType.isInstance(decodedPayload)) { return decodedPayload; } return FastBeanCopier.DEFAULT_CONVERT.convert(decodedPayload, paramType, resolvableType.resolveGenerics()); } finally { - message.release(); + ReferenceCountUtil.safeRelease(message); } } @Override public Mono onMessage(TopicPayload message) { try { - boolean paramVoid = paramType == Void.class; - try { - Object val = proxy.apply(target, paramVoid ? null : convert(message)); - if (val instanceof Publisher) { - return Mono.from((Publisher) val).then(); - } - return Mono.empty(); - } finally { - if (paramVoid) { - message.release(); - } + Object val = proxy.apply(target, paramIsVoid ? null : convert(message)); + if (val instanceof Publisher) { + return Mono.from((Publisher) val).then(); } + return Mono.empty(); } catch (Throwable e) { - log.error("invoke event listener [{}] error", toString(), e); + return Mono.error(e); } - return Mono.empty(); } @Override diff --git a/jetlinks-components/gateway-component/src/main/java/org/jetlinks/community/gateway/spring/SpringMessageBroker.java b/jetlinks-components/gateway-component/src/main/java/org/jetlinks/community/gateway/spring/SpringMessageBroker.java index 6e4537d0..ff5063df 100644 --- a/jetlinks-components/gateway-component/src/main/java/org/jetlinks/community/gateway/spring/SpringMessageBroker.java +++ b/jetlinks-components/gateway-component/src/main/java/org/jetlinks/community/gateway/spring/SpringMessageBroker.java @@ -1,40 +1,51 @@ package org.jetlinks.community.gateway.spring; -import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.hswebframework.web.logger.ReactiveLogger; import org.hswebframework.web.utils.TemplateParser; -import org.jetlinks.community.gateway.annotation.Subscribe; import org.jetlinks.core.event.EventBus; import org.jetlinks.core.event.Subscription; +import org.jetlinks.core.event.TopicPayload; +import org.jetlinks.core.lang.SharedPathString; import org.jetlinks.core.trace.MonoTracer; -import org.jetlinks.core.utils.TopicUtils; +import org.jetlinks.community.gateway.annotation.Subscribe; import org.springframework.beans.BeansException; +import org.springframework.beans.factory.SmartInitializingSingleton; import org.springframework.beans.factory.config.BeanPostProcessor; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.core.Ordered; import org.springframework.core.annotation.AnnotatedElementUtils; import org.springframework.core.annotation.AnnotationAttributes; -import org.springframework.core.env.Environment; -import org.springframework.stereotype.Component; +import org.springframework.core.annotation.Order; import org.springframework.util.ClassUtils; import org.springframework.util.CollectionUtils; import org.springframework.util.ReflectionUtils; import org.springframework.util.StringUtils; import reactor.core.publisher.Mono; -import reactor.core.publisher.Signal; import javax.annotation.Nonnull; +import java.lang.reflect.Method; +import java.util.ArrayList; import java.util.Arrays; -import java.util.function.Consumer; +import java.util.List; +import java.util.function.Function; import java.util.stream.Collectors; -@Component @Slf4j -@AllArgsConstructor -public class SpringMessageBroker implements BeanPostProcessor { +@Order(Ordered.HIGHEST_PRECEDENCE) +public class SpringMessageBroker implements BeanPostProcessor, ApplicationContextAware, SmartInitializingSingleton { - private final EventBus eventBus; + private ApplicationContext context; - private final Environment environment; + private final List dispatchers = new ArrayList<>(); + + public SpringMessageBroker(ApplicationContext context) { + this.context = context; + } + + public SpringMessageBroker() { + + } @Override public Object postProcessAfterInitialization(@Nonnull Object bean, @Nonnull String beanName) throws BeansException { @@ -44,13 +55,34 @@ public class SpringMessageBroker implements BeanPostProcessor { if (CollectionUtils.isEmpty(subscribes)) { return; } + dispatchers.add(new Dispatcher(bean, type, method, context, subscribes)); + }); + + return bean; + } + + private static class Dispatcher implements Function> { + private final ApplicationContext context; + private final ProxyMessageListener listener; + + private final Subscription subscription; + + private final MonoTracer tracer; + private final String callName; + + public Dispatcher(Object bean, + Class type, + Method method, + ApplicationContext context, + AnnotationAttributes subscribes) { + this.context = context; String id = subscribes.getString("id"); if (!StringUtils.hasText(id)) { id = type.getSimpleName().concat(".").concat(method.getName()); } String traceName = "/java/" + type.getSimpleName() + "/" + method.getName(); - String callName = type.getSimpleName() + "." + method.getName(); - Subscription subscription = Subscription + callName = type.getSimpleName() + "." + method.getName(); + subscription = Subscription .builder() .subscriberId("spring:" + id) .topics(Arrays.stream(subscribes.getStringArray("value")) @@ -59,44 +91,62 @@ public class SpringMessageBroker implements BeanPostProcessor { .priority(subscribes.getNumber("priority")) .features((Subscription.Feature[]) subscribes.get("features")) .build(); - - ProxyMessageListener listener = new ProxyMessageListener(bean, method); - - Consumer logError = error -> log.error("handle[{}] event message error : {}", listener, error.getLocalizedMessage(), error); - - MonoTracer tracer = MonoTracer.create(traceName); - - eventBus - .subscribe(subscription, msg -> { - try { - return listener - .onMessage(msg) - .as(tracer) - .doOnError(logError) - .checkpoint(callName); - } catch (Throwable e) { - logError.accept(e); - } - return Mono.empty(); - }); - - }); - - return bean; - } - - protected String convertTopic(String topic) { - if (!topic.contains("${")) { - return topic; + listener = new ProxyMessageListener(bean, method); + tracer = MonoTracer.create(SharedPathString.of(traceName)); } - return TemplateParser.parse(topic, template -> { - String[] arr = template.split(":", 2); - String property = environment.getProperty(arr[0], arr.length > 1 ? arr[1] : ""); - if (StringUtils.isEmpty(property)) { - throw new IllegalArgumentException("Parse topic [" + template + "] error, can not get property : " + arr[0]); + + + protected String convertTopic(String topic) { + if (!topic.contains("${")) { + return topic; } - return property; - }); + return TemplateParser.parse(topic, template -> { + String[] arr = template.split(":", 2); + String property = context.getEnvironment().getProperty(arr[0], arr.length > 1 ? arr[1] : ""); + if (!StringUtils.hasText(property)) { + throw new IllegalArgumentException("Parse topic [" + template + "] error, can not get property : " + arr[0]); + } + return property; + }); + } + + private void logError(Throwable err) { + if(err instanceof BeansException){ + return; + } + log.error("handle[{}] event message error : {}", listener, err.getLocalizedMessage(), err); + } + + void start() { + context + .getBean(EventBus.class) + .subscribe(subscription, this); + } + + @Override + public Mono apply(TopicPayload msg) { + try { + return listener + .onMessage(msg) + .as(tracer) + .doOnError(this::logError) + .checkpoint(callName); + } catch (Throwable e) { + logError(e); + } + return Mono.empty(); + } } + @Override + public void setApplicationContext(@Nonnull ApplicationContext applicationContext) throws BeansException { + this.context = applicationContext; + } + + @Override + public void afterSingletonsInstantiated() { + for (Dispatcher dispatcher : dispatchers) { + dispatcher.start(); + } + } } diff --git a/jetlinks-components/gateway-component/src/main/java/org/jetlinks/community/gateway/supports/DeviceGatewayProperties.java b/jetlinks-components/gateway-component/src/main/java/org/jetlinks/community/gateway/supports/DeviceGatewayProperties.java index ace559bd..0447aad4 100644 --- a/jetlinks-components/gateway-component/src/main/java/org/jetlinks/community/gateway/supports/DeviceGatewayProperties.java +++ b/jetlinks-components/gateway-component/src/main/java/org/jetlinks/community/gateway/supports/DeviceGatewayProperties.java @@ -7,7 +7,7 @@ import lombok.Setter; import org.jetlinks.core.ProtocolSupport; import org.jetlinks.community.ValueObject; -import javax.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotBlank; import java.util.HashMap; import java.util.Map; diff --git a/jetlinks-components/gateway-component/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/jetlinks-components/gateway-component/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports index 17342222..22b39578 100644 --- a/jetlinks-components/gateway-component/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports +++ b/jetlinks-components/gateway-component/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports @@ -1 +1,2 @@ -org.jetlinks.community.gateway.GatewayConfiguration \ No newline at end of file +org.jetlinks.community.gateway.GatewayConfiguration +org.jetlinks.community.gateway.spring.EventBusDispatcherConfiguration \ No newline at end of file diff --git a/jetlinks-components/io-component/pom.xml b/jetlinks-components/io-component/pom.xml index 01756abf..89005846 100644 --- a/jetlinks-components/io-component/pom.xml +++ b/jetlinks-components/io-component/pom.xml @@ -5,7 +5,7 @@ jetlinks-components org.jetlinks.community - 2.3.0-SNAPSHOT + 2.10.0-SNAPSHOT ../pom.xml diff --git a/jetlinks-components/logging-component/pom.xml b/jetlinks-components/logging-component/pom.xml index 00942931..0ec5acdf 100644 --- a/jetlinks-components/logging-component/pom.xml +++ b/jetlinks-components/logging-component/pom.xml @@ -5,7 +5,7 @@ jetlinks-components org.jetlinks.community - 2.3.0-SNAPSHOT + 2.10.0-SNAPSHOT ../pom.xml 4.0.0 @@ -22,7 +22,7 @@ org.jetlinks.community - elasticsearch-component + timeseries-component ${project.version} diff --git a/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/access/AccessLoggerService.java b/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/access/AccessLoggerService.java new file mode 100755 index 00000000..64704557 --- /dev/null +++ b/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/access/AccessLoggerService.java @@ -0,0 +1,40 @@ +package org.jetlinks.community.logging.access; + + +import org.hswebframework.web.api.crud.entity.PagerResult; +import org.hswebframework.web.api.crud.entity.QueryParamEntity; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +/** + * 访问日志服务,用于记录和查询平台的接口访问日志 + * + * @version 2.3 + **/ +public interface AccessLoggerService { + + /** + * 保存访问日志 + * + * @param log 日志内容 + * @return void + */ + Mono save(SerializableAccessLog log); + + /** + * 分页查询访问日志 + * + * @param queryParam 动态查询参数 + * @return 查询结果 + */ + Mono> query(QueryParamEntity queryParam); + + /** + * 不分页查询访问日志 + * + * @param queryParam 查询参数 + * @return 查询结果 + */ + Flux queryNoPaging(QueryParamEntity queryParam); + +} diff --git a/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/access/AccessLoggingTranslator.java b/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/access/AccessLoggingTranslator.java old mode 100644 new mode 100755 index 371a0579..16fba4b5 --- a/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/access/AccessLoggingTranslator.java +++ b/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/access/AccessLoggingTranslator.java @@ -1,13 +1,13 @@ package org.jetlinks.community.logging.access; import org.hswebframework.web.logging.events.AccessLoggerAfterEvent; -import org.jetlinks.community.logging.configuration.LoggingProperties; import org.jetlinks.core.utils.TopicUtils; +import org.jetlinks.community.logging.configuration.LoggingProperties; +import org.jetlinks.community.logging.utils.LoggingUtil; import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.event.EventListener; -import org.springframework.stereotype.Component; +import reactor.core.publisher.Mono; -@Component public class AccessLoggingTranslator { private final ApplicationEventPublisher eventPublisher; @@ -21,12 +21,16 @@ public class AccessLoggingTranslator { @EventListener public void translate(AccessLoggerAfterEvent event) { + for (String pathExclude : properties.getAccess().getPathExcludes()) { if (TopicUtils.match(pathExclude, event.getLogger().getUrl())) { return; } } - eventPublisher.publishEvent(SerializableAccessLog.of(event.getLogger())); + SerializableAccessLog log = SerializableAccessLog.of(event.getLogger()); + + eventPublisher.publishEvent(log); } + } diff --git a/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/access/SerializableAccessLog.java b/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/access/SerializableAccessLog.java old mode 100644 new mode 100755 index 7d3db578..05d3645a --- a/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/access/SerializableAccessLog.java +++ b/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/access/SerializableAccessLog.java @@ -1,23 +1,23 @@ package org.jetlinks.community.logging.access; -import com.alibaba.fastjson.JSON; import io.swagger.v3.oas.annotations.media.Schema; import lombok.Getter; import lombok.Setter; -import org.hswebframework.utils.StringUtils; import org.hswebframework.web.bean.FastBeanCopier; import org.hswebframework.web.logging.AccessLogger; import org.hswebframework.web.logging.AccessLoggerInfo; +import org.jetlinks.community.logging.utils.LoggingUtil; +import org.jetlinks.core.utils.ExceptionUtils; import org.springframework.http.HttpHeaders; -import org.springframework.http.codec.multipart.FilePart; import java.io.Serializable; import java.util.HashMap; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; /** - * @see org.hswebframework.web.logging.AccessLoggerInfo + * @see AccessLoggerInfo */ @Setter @Getter @@ -66,7 +66,7 @@ public class SerializableAccessLog implements Serializable { /** * 请求者ip地址 */ - @Schema(description = "请求中IP") + @Schema(description = "请求者IP") private String ip; /** @@ -117,32 +117,37 @@ public class SerializableAccessLog implements Serializable { @Schema(description = "异常栈信息") private String exception; + private String traceId; + + private String spanId; + + private Set bindings; + + private String creatorId; + + private String ipRegion; + public static SerializableAccessLog of(AccessLoggerInfo info) { SerializableAccessLog accessLog = FastBeanCopier.copy(info, new SerializableAccessLog(), "parameters", "method", "target", "exception"); accessLog.setMethod(info.getMethod().getName()); accessLog.setTarget(info.getTarget().getName()); + //移除敏感请求头 accessLog.getHttpHeaders().remove("X_Access_Token"); + accessLog.getHttpHeaders().remove("X-Access-Token"); accessLog.getHttpHeaders().remove(HttpHeaders.AUTHORIZATION); - accessLog.setException(info.getException() == null ? "" - : StringUtils.throwable2String(info.getException())); - Map newParameter = info.getParameters() + accessLog.setException(info.getException() == null ? "" : ExceptionUtils.getStackTrace(info.getException())); + Map newParameter = info + .getParameters() .entrySet() .stream() - .collect(Collectors.toMap(Map.Entry::getKey, e -> { - Object value = e.getValue(); - if (value instanceof FilePart) { - return ("file:") + ((FilePart) value).filename(); - } - String className = value.getClass().getName(); - if (className.startsWith("org.springframework")) { - return className; - } - return JSON.toJSONString(value); - })); + .collect(Collectors.toMap(Map.Entry::getKey, + e -> LoggingUtil.convertParameterValue(e.getValue()))); accessLog.setParameters(newParameter); return accessLog; } + + } diff --git a/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/access/TimeSeriesAccessLoggerService.java b/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/access/TimeSeriesAccessLoggerService.java new file mode 100644 index 00000000..1e89c717 --- /dev/null +++ b/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/access/TimeSeriesAccessLoggerService.java @@ -0,0 +1,90 @@ +package org.jetlinks.community.logging.access; + +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.hswebframework.web.api.crud.entity.PagerResult; +import org.hswebframework.web.api.crud.entity.QueryParamEntity; +import org.hswebframework.web.bean.FastBeanCopier; +import org.jetlinks.core.metadata.types.ArrayType; +import org.jetlinks.core.metadata.types.DateTimeType; +import org.jetlinks.core.metadata.types.ObjectType; +import org.jetlinks.core.metadata.types.StringType; +import org.jetlinks.community.ConfigMetadataConstants; +import org.jetlinks.community.timeseries.*; +import org.springframework.beans.factory.SmartInitializingSingleton; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.util.HashMap; +import java.util.Map; + +import static org.jetlinks.core.metadata.SimplePropertyMetadata.of; + +@Slf4j +@AllArgsConstructor +public class TimeSeriesAccessLoggerService implements AccessLoggerService, SmartInitializingSingleton { + + public static final TimeSeriesMetric metric = TimeSeriesMetric.of("access_logger"); + + private final TimeSeriesManager timeSeriesManager; + + + @Override + public Mono save(SerializableAccessLog log) { + Map data = FastBeanCopier.copy(log, new HashMap<>()); + + return timeSeriesManager + .getService(metric) + .commit(TimeSeriesData.of(log.getRequestTime(), data)); + } + + @Override + public Mono> query(QueryParamEntity queryParam) { + return timeSeriesManager + .getService(metric) + .queryPager(queryParam, ts -> FastBeanCopier.copy(ts.getData(), new SerializableAccessLog())); + } + + @Override + public Flux queryNoPaging(QueryParamEntity queryParam) { + return timeSeriesManager + .getService(metric) + .query(queryParam) + .map(ts -> FastBeanCopier.copy(ts.getData(), new SerializableAccessLog())); + } + + @Override + public void afterSingletonsInstantiated() { + timeSeriesManager + .registerMetadata( + TimeSeriesMetadata.of( + metric, + of("requestTime", "请求时间", DateTimeType.GLOBAL), + of("responseTime", "响应时间", DateTimeType.GLOBAL), + of("target", "请求类", StringType.GLOBAL), + of("method", "请求方法", StringType.GLOBAL), + of("parameters", "参数", new ObjectType()), + of("action", "操作", StringType.GLOBAL), + + of("ip", "IP地址", StringType.GLOBAL), + of("ipRegion", "IP属地", StringType.GLOBAL), + of("url", "请求地址", StringType.GLOBAL), + of("httpMethod", "HTTP方法", StringType.GLOBAL), + of("httpHeaders", "请求头", new ObjectType()), + + of("exception", "异常信息", new StringType().expand(ConfigMetadataConstants.maxLength, 5120L)), + + of("bindings", "绑定信息", new ArrayType().elementType(StringType.GLOBAL)), + of("creatorId", "创建人", StringType.GLOBAL), + of("spanId", "链路跨度ID", StringType.GLOBAL), + of("traceId", "链路ID", StringType.GLOBAL), + of("context", "上下文", new ObjectType() + .addProperty("userId", "用户ID", StringType.GLOBAL) + .addProperty("username", "用户名", StringType.GLOBAL) + ) + ) + ).subscribe(ignore -> { + }, + error -> log.warn("register access logger metadata error", error)); + } +} diff --git a/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/configuration/LoggingConfiguration.java b/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/configuration/LoggingConfiguration.java index 44d985f4..89d85118 100644 --- a/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/configuration/LoggingConfiguration.java +++ b/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/configuration/LoggingConfiguration.java @@ -1,27 +1,58 @@ package org.jetlinks.community.logging.configuration; import lombok.extern.slf4j.Slf4j; +import org.jetlinks.community.logging.access.AccessLoggerService; +import org.jetlinks.community.logging.access.AccessLoggingTranslator; +import org.jetlinks.community.logging.access.TimeSeriesAccessLoggerService; +import org.jetlinks.community.logging.event.handler.AccessLoggerEventHandler; +import org.jetlinks.community.logging.event.handler.SystemLoggerEventHandler; import org.jetlinks.community.logging.logback.SystemLoggingAppender; +import org.jetlinks.community.logging.system.SystemLoggerService; +import org.jetlinks.community.logging.system.TimeSeriesSystemLoggerService; +import org.jetlinks.community.timeseries.TimeSeriesManager; +import org.springframework.boot.autoconfigure.AutoConfiguration; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.ApplicationEventPublisher; -import org.springframework.context.ApplicationEventPublisherAware; -import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Bean; -@Configuration +@AutoConfiguration @EnableConfigurationProperties(LoggingProperties.class) @Slf4j -public class LoggingConfiguration implements ApplicationEventPublisherAware { +public class LoggingConfiguration { - private final LoggingProperties properties; - - public LoggingConfiguration(LoggingProperties properties) { - this.properties = properties; + @Bean + public AccessLoggingTranslator accessLoggingTranslator(ApplicationEventPublisher eventPublisher, + LoggingProperties properties) { + SystemLoggingAppender.publisher = eventPublisher; SystemLoggingAppender.staticContext.putAll(properties.getSystem().getContext()); + + return new AccessLoggingTranslator(eventPublisher, properties); } - @Override - public void setApplicationEventPublisher(ApplicationEventPublisher applicationEventPublisher) { - SystemLoggingAppender.publisher = applicationEventPublisher; + @Bean + @ConditionalOnMissingBean(AccessLoggerService.class) + public TimeSeriesAccessLoggerService accessLoggerService(TimeSeriesManager timeSeriesManager) { + return new TimeSeriesAccessLoggerService(timeSeriesManager); } + + @Bean + @ConditionalOnMissingBean(SystemLoggerService.class) + public TimeSeriesSystemLoggerService systemLoggerService(TimeSeriesManager timeSeriesManager) { + return new TimeSeriesSystemLoggerService(timeSeriesManager); + } + + + @Bean + public AccessLoggerEventHandler accessLoggerEventHandler(AccessLoggerService loggerService) { + return new AccessLoggerEventHandler(loggerService); + } + + + @Bean + public SystemLoggerEventHandler systemLoggerEventHandler(SystemLoggerService loggerService) { + return new SystemLoggerEventHandler(loggerService); + } + } diff --git a/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/event/handler/AccessLoggerEventHandler.java b/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/event/handler/AccessLoggerEventHandler.java old mode 100644 new mode 100755 index 6fb12c62..6c0d4039 --- a/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/event/handler/AccessLoggerEventHandler.java +++ b/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/event/handler/AccessLoggerEventHandler.java @@ -1,52 +1,29 @@ package org.jetlinks.community.logging.event.handler; import lombok.extern.slf4j.Slf4j; -import org.jetlinks.community.elastic.search.index.DefaultElasticSearchIndexMetadata; -import org.jetlinks.community.elastic.search.index.ElasticSearchIndexManager; -import org.jetlinks.community.elastic.search.service.ElasticSearchService; +import org.jetlinks.community.logging.access.AccessLoggerService; import org.jetlinks.community.logging.access.SerializableAccessLog; -import org.jetlinks.core.metadata.types.DateTimeType; -import org.jetlinks.core.metadata.types.ObjectType; -import org.jetlinks.core.metadata.types.StringType; import org.springframework.context.event.EventListener; import org.springframework.core.annotation.Order; -import org.springframework.stereotype.Component; -import reactor.core.publisher.Mono; /** * @author bsetfeng * @since 1.0 **/ -@Component @Slf4j @Order(5) public class AccessLoggerEventHandler { - private final ElasticSearchService elasticSearchService; - - - public AccessLoggerEventHandler(ElasticSearchService elasticSearchService, ElasticSearchIndexManager indexManager) { - this.elasticSearchService = elasticSearchService; - indexManager.putIndex( - new DefaultElasticSearchIndexMetadata(LoggerIndexProvider.ACCESS.getIndex()) - .addProperty("requestTime", new DateTimeType()) - .addProperty("responseTime", new DateTimeType()) - .addProperty("action", new StringType()) - .addProperty("ip", new StringType()) - .addProperty("url", new StringType()) - .addProperty("httpHeaders", new ObjectType()) - .addProperty("context", new ObjectType() - .addProperty("userId",new StringType()) - .addProperty("username",new StringType()) - ) - ).subscribe(); + private final AccessLoggerService loggerService; + public AccessLoggerEventHandler(AccessLoggerService loggerService) { + this.loggerService = loggerService; } @EventListener public void acceptAccessLoggerInfo(SerializableAccessLog info) { - elasticSearchService.commit(LoggerIndexProvider.ACCESS, Mono.just(info)).subscribe(); + loggerService.save(info).subscribe(); } diff --git a/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/event/handler/LoggerIndexProvider.java b/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/event/handler/LoggerIndexProvider.java deleted file mode 100644 index af454a8c..00000000 --- a/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/event/handler/LoggerIndexProvider.java +++ /dev/null @@ -1,19 +0,0 @@ -package org.jetlinks.community.logging.event.handler; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import org.jetlinks.community.elastic.search.index.ElasticIndex; - -/** - * @author bsetfeng - * @since 1.0 - **/ -@Getter -@AllArgsConstructor -public enum LoggerIndexProvider implements ElasticIndex { - - ACCESS("access_logger"), - SYSTEM("system_logger"); - - private String index; -} diff --git a/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/event/handler/SystemLoggerEventHandler.java b/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/event/handler/SystemLoggerEventHandler.java index d4ba82e7..df475f85 100644 --- a/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/event/handler/SystemLoggerEventHandler.java +++ b/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/event/handler/SystemLoggerEventHandler.java @@ -1,18 +1,11 @@ package org.jetlinks.community.logging.event.handler; import lombok.extern.slf4j.Slf4j; -import org.jetlinks.community.elastic.search.index.DefaultElasticSearchIndexMetadata; -import org.jetlinks.community.elastic.search.index.ElasticSearchIndexManager; -import org.jetlinks.community.elastic.search.service.ElasticSearchService; import org.jetlinks.community.logging.system.SerializableSystemLog; -import org.jetlinks.core.event.EventBus; -import org.jetlinks.core.metadata.types.DateTimeType; -import org.jetlinks.core.metadata.types.ObjectType; -import org.jetlinks.core.metadata.types.StringType; +import org.jetlinks.community.logging.system.SystemLoggerService; import org.springframework.context.event.EventListener; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; -import reactor.core.publisher.Mono; /** * @author bsetfeng @@ -23,41 +16,18 @@ import reactor.core.publisher.Mono; @Order(5) public class SystemLoggerEventHandler { - private final EventBus eventBus; + private final SystemLoggerService loggerService; - private final ElasticSearchService elasticSearchService; - - public SystemLoggerEventHandler(ElasticSearchService elasticSearchService, - ElasticSearchIndexManager indexManager, - EventBus eventBus) { - this.elasticSearchService = elasticSearchService; - this.eventBus = eventBus; - - indexManager.putIndex( - new DefaultElasticSearchIndexMetadata(LoggerIndexProvider.SYSTEM.getIndex()) - .addProperty("createTime", new DateTimeType()) - .addProperty("name", new StringType()) - .addProperty("level", new StringType()) - .addProperty("message", new StringType()) - .addProperty("className",new StringType()) - .addProperty("exceptionStack",new StringType()) - .addProperty("methodName",new StringType()) - .addProperty("threadId",new StringType()) - .addProperty("threadName",new StringType()) - .addProperty("id",new StringType()) - .addProperty("context", new ObjectType() - .addProperty("requestId",new StringType()) - .addProperty("server",new StringType())) - ).subscribe(); + public SystemLoggerEventHandler(SystemLoggerService loggerService) { + this.loggerService = loggerService; } @EventListener public void acceptAccessLoggerInfo(SerializableSystemLog info) { - eventBus - .publish("/logging/system/" + info.getName().replace(".", "/") + "/" + (info.getLevel().toLowerCase()), info) - .subscribe(); - elasticSearchService.commit(LoggerIndexProvider.SYSTEM, Mono.just(info)) + loggerService + .save(info) .subscribe(); } + } diff --git a/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/logback/ShortenedThrowableConverter.java b/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/logback/ShortenedThrowableConverter.java new file mode 100644 index 00000000..ade2898d --- /dev/null +++ b/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/logback/ShortenedThrowableConverter.java @@ -0,0 +1,82 @@ +package org.jetlinks.community.logging.logback; + +import ch.qos.logback.classic.pattern.ThrowableHandlingConverter; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.classic.spi.IThrowableProxy; +import ch.qos.logback.classic.spi.StackTraceElementProxy; +import ch.qos.logback.classic.spi.ThrowableProxy; +import org.jetlinks.core.utils.ExceptionUtils; + +public class ShortenedThrowableConverter extends ThrowableHandlingConverter { + @Override + public String convert(ILoggingEvent event) { + return getStackTrace(event.getThrowableProxy()); + } + + public static void writeStackTraceElement(StringBuilder builder, + StackTraceElementProxy[] elements) { + int unimportantCount = 0; + for (StackTraceElementProxy element : elements) { + if (ExceptionUtils.compactEnabled && ExceptionUtils.isUnimportant(element.getStackTraceElement())) { + unimportantCount++; + continue; + } + if (unimportantCount > 0) { + builder.append("\t...") + .append(unimportantCount) + .append(" frames excluded\n"); + unimportantCount = 0; + } + builder.append("\t") + .append(element) + .append("\n"); + } + + if (unimportantCount > 0) { + builder.append("\t...") + .append(unimportantCount) + .append(" frames excluded\n"); + } + } + + public static String getStackTrace(IThrowableProxy e) { + if (e == null) { + return ""; + } + return getStackTrace(new StringBuilder(), e).toString(); + } + + public static StringBuilder getStackTrace(StringBuilder builder, + IThrowableProxy e) { + if (e instanceof ThrowableProxy) { + builder + .append(((ThrowableProxy) e).getThrowable()); + } else { + builder + .append(e.getClassName()) + .append(e.getMessage()); + } + + + builder.append("\n"); + + StackTraceElementProxy[] elements = e.getStackTraceElementProxyArray(); + if (elements != null && elements.length != 0) { + writeStackTraceElement(builder, elements); + } + + for (IThrowableProxy throwable : e.getSuppressed()) { + builder.append("Suppressed: "); + getStackTrace(builder, throwable); + } + + IThrowableProxy cause = e.getCause(); + if (cause != null) { + builder.append("Caused by: "); + getStackTrace(builder, cause); + } + + return builder; + } + +} diff --git a/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/logback/SystemLoggingAppender.java b/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/logback/SystemLoggingAppender.java index d56e5d4f..d17cfb3e 100644 --- a/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/logback/SystemLoggingAppender.java +++ b/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/logback/SystemLoggingAppender.java @@ -2,18 +2,14 @@ package org.jetlinks.community.logging.logback; import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.classic.spi.IThrowableProxy; -import ch.qos.logback.classic.spi.StackTraceElementProxy; -import ch.qos.logback.classic.spi.ThrowableProxyUtil; -import ch.qos.logback.core.CoreConstants; import ch.qos.logback.core.UnsynchronizedAppenderBase; +import lombok.Generated; import lombok.extern.slf4j.Slf4j; import org.hswebframework.web.id.IDGenerator; -import org.hswebframework.web.utils.ModuleUtils; import org.jetlinks.community.logging.system.SerializableSystemLog; import org.slf4j.MDC; +import org.springframework.beans.factory.BeanCreationNotAllowedException; import org.springframework.context.ApplicationEventPublisher; -import org.springframework.util.ClassUtils; -import org.springframework.util.StringUtils; import java.util.*; import java.util.concurrent.ConcurrentHashMap; @@ -26,89 +22,52 @@ public class SystemLoggingAppender extends UnsynchronizedAppenderBase staticContext = new ConcurrentHashMap<>(); @Override + @Generated protected void append(ILoggingEvent event) { + doPublish(convertLog(event)); + } + protected static void doPublish(SerializableSystemLog systemLog) { if (publisher == null) { return; } + try { + publisher.publishEvent(systemLog); + } catch (IllegalStateException | BeanCreationNotAllowedException ignore) { + } catch (Throwable e) { + log.error("publish system log error", e); + } + } + + static SerializableSystemLog convertLog(ILoggingEvent event) { StackTraceElement element = event.getCallerData()[0]; IThrowableProxy proxies = event.getThrowableProxy(); String message = event.getFormattedMessage(); - String stack; - StringJoiner joiner = new StringJoiner("\n", message + "\n[", "]"); - Queue queue = new LinkedList<>(); - queue.add(proxies); - while (queue.size() > 0) { - IThrowableProxy proxy = queue.poll(); - if(proxy==null){ - break; - } - int commonFrames = proxy.getCommonFrames(); - StackTraceElementProxy[] stepArray = proxy.getStackTraceElementProxyArray(); + String stack = ShortenedThrowableConverter.getStackTrace(proxies); - StringBuilder stringBuilder = new StringBuilder(); - ThrowableProxyUtil.subjoinFirstLine(stringBuilder, proxy); - joiner.add(stringBuilder); - for (int i = 0; i < stepArray.length - commonFrames; i++) { - StringBuilder sb = new StringBuilder(); - sb.append(CoreConstants.TAB); - ThrowableProxyUtil.subjoinSTEP(sb, stepArray[i]); - joiner.add(sb); - } - queue.addAll(Arrays.asList(proxy.getSuppressed())); + Map context = new HashMap<>(staticContext); + Map mdc = MDC.getCopyOfContextMap(); + if (mdc != null) { + context.putAll(mdc); } - stack = joiner.toString(); + SerializableSystemLog info = SerializableSystemLog + .builder() + .id(IDGenerator.RANDOM.generate()) + .context(context) + .name(event.getLoggerName()) + .level(event.getLevel().levelStr) + .className(element.getClassName()) + .methodName(element.getMethodName()) + .lineNumber(element.getLineNumber()) + .exceptionStack(stack) + .threadName(event.getThreadName()) + .createTime(event.getTimeStamp()) + .message(message) + .threadId(String.valueOf(Thread.currentThread().getId())) + .build(); - try { - String gitLocation = null; - String mavenModule = null; - try { - Class clazz = Class.forName(element.getClassName()); - ModuleUtils.ModuleInfo moduleInfo = ModuleUtils.getModuleByClass(clazz); - if (!StringUtils.isEmpty(moduleInfo.getGitRepository())) { - StringBuilder javaSb = new StringBuilder(); - javaSb.append(moduleInfo.getGitLocation()); - javaSb.append("src/main/java/"); - javaSb.append((ClassUtils.getPackageName(Class.forName(element.getClassName())).replace(".", "/"))); - javaSb.append("/"); - javaSb.append(Class.forName(element.getClassName()).getSimpleName()); - javaSb.append(".java#L"); - javaSb.append(element.getLineNumber()); - gitLocation = javaSb.toString(); - } - mavenModule = moduleInfo.getArtifactId(); - } catch (Exception ignore) { - - } - Map context = new HashMap<>(staticContext); - Map mdc = MDC.getCopyOfContextMap(); - if (mdc != null) { - context.putAll(mdc); - } - SerializableSystemLog info = SerializableSystemLog.builder() - .id(IDGenerator.RANDOM.generate()) - .mavenModule(mavenModule) - .context(context) - .name(event.getLoggerName()) - .level(event.getLevel().levelStr) - .className(element.getClassName()) - .methodName(element.getMethodName()) - .lineNumber(element.getLineNumber()) - .exceptionStack(stack) - .java(gitLocation) - .threadName(event.getThreadName()) - .createTime(event.getTimeStamp()) - .message(message) - .threadId(String.valueOf(Thread.currentThread().getId())) - .build(); - try { - publisher.publishEvent(info); - } catch (Exception ignore) { - } - } catch (Exception e) { - log.error("组装系统日志错误", e); - } + return info; } } diff --git a/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/system/SerializableSystemLog.java b/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/system/SerializableSystemLog.java old mode 100644 new mode 100755 index b2995f88..cfa7749f --- a/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/system/SerializableSystemLog.java +++ b/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/system/SerializableSystemLog.java @@ -11,6 +11,7 @@ import java.util.Map; @Setter @NoArgsConstructor @AllArgsConstructor +@Generated public class SerializableSystemLog implements Serializable { @Schema(description = "ID") @@ -54,4 +55,10 @@ public class SerializableSystemLog implements Serializable { @Schema(description = "上下文") private Map context; + + @Schema(description = "链路ID") + private String traceId; + + @Schema(description = "链路跨度ID") + private String spanId; } diff --git a/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/system/SystemLoggerService.java b/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/system/SystemLoggerService.java new file mode 100755 index 00000000..8ebc8015 --- /dev/null +++ b/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/system/SystemLoggerService.java @@ -0,0 +1,40 @@ +package org.jetlinks.community.logging.system; + + +import org.hswebframework.web.api.crud.entity.PagerResult; +import org.hswebframework.web.api.crud.entity.QueryParamEntity; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +/** + * 系统日志服务,用于记录和查询系统内部运行过程中产生的日志 + * + * @version 2.3 + * @see org.jetlinks.community.logging.logback.SystemLoggingAppender + **/ +public interface SystemLoggerService { + + /** + * 保存系统日志 + * + * @param log 日志内容 + * @return void + */ + Mono save(SerializableSystemLog log); + + /** + * 分页查询系统日志 + * + * @param queryParam 动态查询参数 + * @return 查询结果 + */ + Mono> query(QueryParamEntity queryParam); + + /** + * 不分页查询系统日志 + * + * @param queryParam 查询参数 + * @return 查询结果 + */ + Flux queryNoPaging(QueryParamEntity queryParam); +} \ No newline at end of file diff --git a/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/system/TimeSeriesSystemLoggerService.java b/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/system/TimeSeriesSystemLoggerService.java new file mode 100644 index 00000000..4e8a0859 --- /dev/null +++ b/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/system/TimeSeriesSystemLoggerService.java @@ -0,0 +1,87 @@ +package org.jetlinks.community.logging.system; + +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.hswebframework.web.api.crud.entity.PagerResult; +import org.hswebframework.web.api.crud.entity.QueryParamEntity; +import org.hswebframework.web.bean.FastBeanCopier; +import org.jetlinks.core.metadata.types.DateTimeType; +import org.jetlinks.core.metadata.types.IntType; +import org.jetlinks.core.metadata.types.ObjectType; +import org.jetlinks.core.metadata.types.StringType; +import org.jetlinks.community.ConfigMetadataConstants; +import org.jetlinks.community.timeseries.TimeSeriesData; +import org.jetlinks.community.timeseries.TimeSeriesManager; +import org.jetlinks.community.timeseries.TimeSeriesMetadata; +import org.jetlinks.community.timeseries.TimeSeriesMetric; +import org.springframework.beans.factory.SmartInitializingSingleton; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.util.HashMap; +import java.util.Map; + +import static org.jetlinks.core.metadata.SimplePropertyMetadata.of; + +@Slf4j +@AllArgsConstructor +public class TimeSeriesSystemLoggerService implements SystemLoggerService, SmartInitializingSingleton { + + public static final TimeSeriesMetric metric = TimeSeriesMetric.of("system_logger"); + + private final TimeSeriesManager timeSeriesManager; + + + @Override + public Mono save(SerializableSystemLog log) { + Map data = FastBeanCopier.copy(log, new HashMap<>()); + + return timeSeriesManager + .getService(metric) + .commit(TimeSeriesData.of(log.getCreateTime(), data)); + } + + @Override + public Mono> query(QueryParamEntity queryParam) { + return timeSeriesManager + .getService(metric) + .queryPager(queryParam, ts -> FastBeanCopier.copy(ts.getData(), new SerializableSystemLog())); + } + + @Override + public Flux queryNoPaging(QueryParamEntity queryParam) { + return timeSeriesManager + .getService(metric) + .query(queryParam) + .map(ts -> FastBeanCopier.copy(ts.getData(), new SerializableSystemLog())); + } + + @Override + public void afterSingletonsInstantiated() { + + timeSeriesManager + .registerMetadata( + TimeSeriesMetadata.of( + metric, + of("createTime", "创建时间", DateTimeType.GLOBAL), + of("name", "日志名", StringType.GLOBAL), + of("level", "日志级别", StringType.GLOBAL), + of("message", "日志内容", new StringType().expand(ConfigMetadataConstants.maxLength, 5120L)), + of("className", "类名", StringType.GLOBAL), + of("exceptionStack", "异常栈", new StringType().expand(ConfigMetadataConstants.maxLength, 5120L)), + of("methodName", "方法名", StringType.GLOBAL), + of("lineNumber", "行号", IntType.GLOBAL), + of("threadId", "线程ID", StringType.GLOBAL), + of("threadName", "线程名称", StringType.GLOBAL), + of("spanId", "链路跨度ID", StringType.GLOBAL), + of("traceId", "链路ID", StringType.GLOBAL), + of("context", "上下文", new ObjectType() + .addProperty("userId", "用户ID", StringType.GLOBAL) + .addProperty("username", "用户名", StringType.GLOBAL) + ) + ) + ).subscribe(ignore -> { + }, + error -> log.warn("register system logger metadata error", error)); + } +} diff --git a/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/utils/LoggingUtil.java b/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/utils/LoggingUtil.java new file mode 100644 index 00000000..432bd2ef --- /dev/null +++ b/jetlinks-components/logging-component/src/main/java/org/jetlinks/community/logging/utils/LoggingUtil.java @@ -0,0 +1,30 @@ +package org.jetlinks.community.logging.utils; + +import org.jetlinks.community.utils.ObjectMappers; +import org.springframework.http.codec.multipart.FilePart; + +/** + * @author gyl + * @since 2.2 + */ +public class LoggingUtil { + + public static Object convertParameterValue(Object value) { + if (value instanceof FilePart) { + return ("file:") + ((FilePart) value).filename(); + } + if (value instanceof Class) { + return ((Class) value).getName(); + } + String className = value.getClass().getName(); + if (className.startsWith("org.springframework")) { + return className; + } + if (value instanceof String || value instanceof Number) { + return value; + } + return ObjectMappers.toJsonString(value); + } + + +} diff --git a/jetlinks-components/logging-component/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/jetlinks-components/logging-component/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports new file mode 100644 index 00000000..72a07ef1 --- /dev/null +++ b/jetlinks-components/logging-component/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports @@ -0,0 +1 @@ +org.jetlinks.community.logging.configuration.LoggingConfiguration \ No newline at end of file diff --git a/jetlinks-components/network-component/http-component/pom.xml b/jetlinks-components/network-component/http-component/pom.xml index d535b5a4..164ac63f 100755 --- a/jetlinks-components/network-component/http-component/pom.xml +++ b/jetlinks-components/network-component/http-component/pom.xml @@ -5,7 +5,7 @@ network-component org.jetlinks.community - 2.3.0-SNAPSHOT + 2.10.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/jetlinks-components/network-component/http-component/src/main/java/org/jetlinks/community/network/http/server/HttpExchange.java b/jetlinks-components/network-component/http-component/src/main/java/org/jetlinks/community/network/http/server/HttpExchange.java index 3c982005..df5cec73 100755 --- a/jetlinks-components/network-component/http-component/src/main/java/org/jetlinks/community/network/http/server/HttpExchange.java +++ b/jetlinks-components/network-component/http-component/src/main/java/org/jetlinks/community/network/http/server/HttpExchange.java @@ -11,7 +11,7 @@ import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import reactor.core.publisher.Mono; -import javax.validation.constraints.NotNull; +import jakarta.validation.constraints.NotNull; /** * HTTP交换接口，支持获取请求和发送响应 diff --git a/jetlinks-components/network-component/mqtt-component/pom.xml b/jetlinks-components/network-component/mqtt-component/pom.xml index 84d9702f..9a5494f9 100644 --- a/jetlinks-components/network-component/mqtt-component/pom.xml +++ b/jetlinks-components/network-component/mqtt-component/pom.xml @@ -5,7 +5,7 @@ network-component org.jetlinks.community - 2.3.0-SNAPSHOT + 2.10.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/jetlinks-components/network-component/network-core/pom.xml b/jetlinks-components/network-component/network-core/pom.xml index b4b832b4..55c486d6 100644 --- a/jetlinks-components/network-component/network-core/pom.xml +++ b/jetlinks-components/network-component/network-core/pom.xml @@ -3,7 +3,7 @@ network-component org.jetlinks.community - 2.3.0-SNAPSHOT + 2.10.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/jetlinks-components/network-component/network-core/src/main/java/org/jetlinks/community/network/AbstractServerNetworkConfig.java b/jetlinks-components/network-component/network-core/src/main/java/org/jetlinks/community/network/AbstractServerNetworkConfig.java index 7d529d8a..c1fa23bd 100644 --- a/jetlinks-components/network-component/network-core/src/main/java/org/jetlinks/community/network/AbstractServerNetworkConfig.java +++ b/jetlinks-components/network-component/network-core/src/main/java/org/jetlinks/community/network/AbstractServerNetworkConfig.java @@ -5,7 +5,7 @@ import lombok.Setter; import org.hibernate.validator.constraints.Range; import org.jetlinks.community.network.resource.NetworkTransport; -import javax.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotBlank; @Getter @Setter diff --git a/jetlinks-components/network-component/network-core/src/main/java/org/jetlinks/community/network/DefaultNetworkManager.java b/jetlinks-components/network-component/network-core/src/main/java/org/jetlinks/community/network/ClusterNetworkManager.java old mode 100644 new mode 100755 similarity index 51% rename from jetlinks-components/network-component/network-core/src/main/java/org/jetlinks/community/network/DefaultNetworkManager.java rename to jetlinks-components/network-component/network-core/src/main/java/org/jetlinks/community/network/ClusterNetworkManager.java index 0a9b38e7..8a2b9788 --- a/jetlinks-components/network-component/network-core/src/main/java/org/jetlinks/community/network/DefaultNetworkManager.java +++ b/jetlinks-components/network-component/network-core/src/main/java/org/jetlinks/community/network/ClusterNetworkManager.java @@ -1,45 +1,41 @@ package org.jetlinks.community.network; -import lombok.*; import lombok.extern.slf4j.Slf4j; +import org.hswebframework.web.crud.utils.TransactionUtils; import org.jetlinks.core.cache.ReactiveCacheContainer; import org.jetlinks.core.event.EventBus; import org.jetlinks.core.event.Subscription; -import org.springframework.beans.BeansException; -import org.springframework.beans.factory.config.BeanPostProcessor; +import org.jetlinks.community.network.channel.Address; import org.springframework.boot.CommandLineRunner; -import org.springframework.stereotype.Component; +import org.springframework.transaction.reactive.TransactionSynchronization; +import reactor.core.Disposable; +import reactor.core.Disposables; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.function.Function3; import javax.annotation.Nonnull; -import javax.annotation.PostConstruct; -import java.io.Serializable; import java.time.Duration; -import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.RejectedExecutionException; -/** - * 默认网络管理器 - * - * @author zhouhao - */ -@Component @Slf4j -public class DefaultNetworkManager implements NetworkManager, BeanPostProcessor, CommandLineRunner { +public class ClusterNetworkManager implements NetworkManager, CommandLineRunner { private final NetworkConfigManager configManager; + private final EventBus eventBus; + private final Map> store = new ConcurrentHashMap<>(); - private final Map> providerSupport = new ConcurrentHashMap<>(); + private final Disposable.Composite disposable = Disposables.composite(); - public DefaultNetworkManager(NetworkConfigManager configManager) { + public ClusterNetworkManager(EventBus eventBus, NetworkConfigManager configManager) { this.configManager = configManager; + this.eventBus = eventBus; } protected void checkNetwork() { @@ -47,7 +43,11 @@ public class DefaultNetworkManager implements NetworkManager, BeanPostProcessor, .flatMapIterable(ReactiveCacheContainer::valuesNow) .filter(i -> !i.isAlive()) .flatMap(network -> { - NetworkProvider provider = providerSupport.get(network.getType().getId()); + @SuppressWarnings("all") + NetworkProvider provider = (NetworkProvider) NetworkProvider + .supports + .get(network.getType().getId()) + .orElse(null); if (provider == null || !network.isAutoReload()) { return Mono.empty(); } @@ -61,7 +61,7 @@ public class DefaultNetworkManager implements NetworkManager, BeanPostProcessor, return Mono.empty(); }); }) - .subscribe(net -> log.info("reloaded network :{}", net)); + .subscribe(net -> log.debug("reloaded network :{}", net)); } private ReactiveCacheContainer getNetworkStore(String type) { @@ -110,7 +110,12 @@ public class DefaultNetworkManager implements NetworkManager, BeanPostProcessor, @SuppressWarnings("all") NetworkProvider networkProvider = (NetworkProvider) this .getProvider(type.getId()) - .orElseThrow(() -> new UnsupportedOperationException("不支持的类型:" + type.getName())); + .orElse(null); + //当前节点不支持此网络组件,则忽略创建. + //在微服务分布式方式部署时可能会出现. + if (networkProvider == null) { + return Mono.empty(); + } return configManager .getConfig(type, id) @@ -120,25 +125,17 @@ public class DefaultNetworkManager implements NetworkManager, BeanPostProcessor, } public void register(NetworkProvider provider) { - this.providerSupport.put(provider.getType().getId(), provider); - } - - @Override - public Object postProcessAfterInitialization(@Nonnull Object bean, @Nonnull String beanName) throws BeansException { - if (bean instanceof NetworkProvider) { - register(((NetworkProvider) bean)); - } - return bean; + NetworkProvider.supports.register(provider.getType().getId(), provider); } @Override public List> getProviders() { - return new ArrayList<>(providerSupport.values()); + return NetworkProvider.supports.getAll(); } @Override public Optional> getProvider(String type) { - return Optional.ofNullable(providerSupport.get(type)); + return NetworkProvider.supports.get(type); } private Mono doReload(String type, String id) { @@ -169,7 +166,18 @@ public class DefaultNetworkManager implements NetworkManager, BeanPostProcessor, @Override public Mono reload(NetworkType type, String id) { - return doReload(type.getId(), id); + return doReload(type.getId(), id) + .then( + TransactionUtils + .registerSynchronization(new TransactionSynchronization() { + @Override + @Nonnull + public Mono afterCommit() { + return eventBus + .publish("/_sys/network/" + type.getId() + "/reload", id) + .then(); + } + }, TransactionSynchronization::afterCommit)); } @@ -177,6 +185,7 @@ public class DefaultNetworkManager implements NetworkManager, BeanPostProcessor, public Mono shutdown(NetworkType type, String id) { return this .doShutdown(type.getId(), id) + .then(eventBus.publish("/_sys/network/" + type.getId() + "/shutdown", id)) .then(); } @@ -184,26 +193,84 @@ public class DefaultNetworkManager implements NetworkManager, BeanPostProcessor, public Mono destroy(NetworkType type, String id) { return this .doDestroy(type.getId(), id) + .then(eventBus.publish("/_sys/network/" + type.getId() + "/destroy", id)) .then(); } + public Flux
getAddresses(NetworkType type, String id,boolean selfServer) { + return configManager + .getConfig(type, id, selfServer) + .flatMap(prop -> this + .getProvider(type.getId()) + .map(provider -> provider.createConfig(prop)) + .orElse(Mono.empty())) + .mapNotNull((conf) -> { + // todo 健康检查 + if (conf instanceof ClientNetworkConfig) { + //客户端则返回远程地址 + return Address.of(((ClientNetworkConfig) conf).getRemoteAddress(), Address.HEALTH_OK); + } + if (conf instanceof ServerNetworkConfig) { + //服务端返回公共访问地址 + return Address.of(((ServerNetworkConfig) conf).getPublicAddress(), Address.HEALTH_OK); + } + return null; + }); + } @Override public void run(String... args) { //定时检查网络组件状态 - Flux.interval(Duration.ofSeconds(10)) - .subscribe(t -> this.checkNetwork()); + disposable.add( + Flux.interval(Duration.ofSeconds(10)) + .subscribe(t -> this.checkNetwork()) + ); + + disposable.add( + eventBus + .subscribe( + Subscription + .builder() + .subscriberId("network-config-manager") + .topics("/_sys/network/*/*") + .justBroker() + .build(), + payload -> { + Map vars = payload.getTopicVars("/_sys/network/{type}/{action}"); + String id = payload.bodyToString(true); + log.debug("{} network [{}-{}]", vars.get("action"), vars.get("type"), id); + if ("reload".equals(vars.get("action"))) { + return this + .doReload(vars.get("type"), id) + .onErrorResume(err -> { + log.error("reload network error", err); + return Mono.empty(); + }); + } + if ("shutdown".equals(vars.get("action"))) { + return doShutdown(vars.get("type"), id); + } + if ("destroy".equals(vars.get("action"))) { + return doDestroy(vars.get("type"), id); + } + return Mono.empty(); + })); } + public void shutdown() { + log.info("shutdown network manager"); + disposable.dispose(); + for (ReactiveCacheContainer value : store.values()) { + for (Network network : value.valuesNow()) { + try { + network.shutdown(); + } catch (RejectedExecutionException ignore) { - @Getter - @Setter - @AllArgsConstructor - @NoArgsConstructor - @Generated - public static class Synchronization implements Serializable { - private NetworkType type; - - private String id; + } catch (Throwable e) { + log.warn("shutdown network error", e); + } + } + } } + } diff --git a/jetlinks-components/network-component/network-core/src/main/java/org/jetlinks/community/network/NetworkConfiguration.java b/jetlinks-components/network-component/network-core/src/main/java/org/jetlinks/community/network/NetworkConfiguration.java index 9b0c57f6..316e0cd4 100644 --- a/jetlinks-components/network-component/network-core/src/main/java/org/jetlinks/community/network/NetworkConfiguration.java +++ b/jetlinks-components/network-component/network-core/src/main/java/org/jetlinks/community/network/NetworkConfiguration.java @@ -5,7 +5,10 @@ import org.jetlinks.community.network.resource.NetworkResourceManager; import org.jetlinks.community.network.resource.NetworkResourceUser; import org.jetlinks.community.network.resource.cluster.DefaultNetworkResourceManager; import org.jetlinks.community.network.resource.cluster.NetworkResourceProperties; +import org.jetlinks.core.event.EventBus; import org.springframework.beans.factory.ObjectProvider; +import org.springframework.boot.CommandLineRunner; +import org.springframework.boot.autoconfigure.AutoConfiguration; import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; @@ -13,7 +16,7 @@ import org.springframework.context.annotation.Configuration; import java.util.stream.Collectors; -@Configuration(proxyBeanMethods = false) +@AutoConfiguration @EnableConfigurationProperties(NetworkResourceProperties.class) public class NetworkConfiguration { @@ -36,4 +39,18 @@ public class NetworkConfiguration { return new DefaultNetworkResourceUser(configManager, networkManager); } + @Bean(destroyMethod = "shutdown") + public ClusterNetworkManager networkManager(EventBus eventBus, + NetworkConfigManager configManager) { + return new ClusterNetworkManager(eventBus, configManager); + } + + @Bean + public CommandLineRunner networkProviderRegister(ObjectProvider> providers) { + for (NetworkProvider provider : providers) { + NetworkProvider.supports.register(provider.getType().getId(), provider); + } + return ignore -> { + }; + } } diff --git a/jetlinks-components/network-component/network-core/src/main/java/org/jetlinks/community/network/NetworkProvider.java b/jetlinks-components/network-component/network-core/src/main/java/org/jetlinks/community/network/NetworkProvider.java index bae1d1ca..b02711b0 100755 --- a/jetlinks-components/network-component/network-core/src/main/java/org/jetlinks/community/network/NetworkProvider.java +++ b/jetlinks-components/network-component/network-core/src/main/java/org/jetlinks/community/network/NetworkProvider.java @@ -1,5 +1,6 @@ package org.jetlinks.community.network; +import org.jetlinks.community.spi.Provider; import org.jetlinks.core.metadata.ConfigMetadata; import reactor.core.publisher.Mono; @@ -12,6 +13,8 @@ import javax.annotation.Nullable; * @param

网络组件类型 */ public interface NetworkProvider

{ + @SuppressWarnings("all") + Provider> supports = Provider.create(NetworkProvider.class); /** * @return 类型 diff --git a/jetlinks-components/network-component/pom.xml b/jetlinks-components/network-component/pom.xml index db96bff1..b920eed2 100644 --- a/jetlinks-components/network-component/pom.xml +++ b/jetlinks-components/network-component/pom.xml @@ -5,7 +5,7 @@ jetlinks-components org.jetlinks.community - 2.3.0-SNAPSHOT + 2.10.0-SNAPSHOT ../pom.xml pom diff --git a/jetlinks-components/network-component/tcp-component/pom.xml b/jetlinks-components/network-component/tcp-component/pom.xml index 09a1d307..9e6262c8 100644 --- a/jetlinks-components/network-component/tcp-component/pom.xml +++ b/jetlinks-components/network-component/tcp-component/pom.xml @@ -5,7 +5,7 @@ network-component org.jetlinks.community - 2.3.0-SNAPSHOT + 2.10.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/jetlinks-components/notify-component/notify-core/pom.xml b/jetlinks-components/notify-component/notify-core/pom.xml index ba0121f0..7563d0b8 100755 --- a/jetlinks-components/notify-component/notify-core/pom.xml +++ b/jetlinks-components/notify-component/notify-core/pom.xml @@ -5,7 +5,7 @@ notify-component org.jetlinks.community - 2.3.0-SNAPSHOT + 2.10.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/jetlinks-components/notify-component/notify-core/src/main/java/org/jetlinks/community/notify/DefaultNotifierManager.java b/jetlinks-components/notify-component/notify-core/src/main/java/org/jetlinks/community/notify/DefaultNotifierManager.java index 18e3877f..c4e68efe 100755 --- a/jetlinks-components/notify-component/notify-core/src/main/java/org/jetlinks/community/notify/DefaultNotifierManager.java +++ b/jetlinks-components/notify-component/notify-core/src/main/java/org/jetlinks/community/notify/DefaultNotifierManager.java @@ -1,11 +1,14 @@ package org.jetlinks.community.notify; import lombok.extern.slf4j.Slf4j; +import org.jetlinks.core.cache.ReactiveCacheContainer; import org.jetlinks.core.event.EventBus; import org.jetlinks.core.event.Subscription; import org.springframework.beans.BeansException; +import org.springframework.beans.factory.SmartInitializingSingleton; import org.springframework.beans.factory.config.BeanPostProcessor; import org.springframework.boot.CommandLineRunner; +import org.springframework.context.ApplicationContext; import org.springframework.stereotype.Component; import reactor.core.publisher.Mono; @@ -14,21 +17,24 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @Slf4j -@Component @SuppressWarnings("all") -public class DefaultNotifierManager implements NotifierManager, BeanPostProcessor, CommandLineRunner { +public class DefaultNotifierManager implements NotifierManager, CommandLineRunner, SmartInitializingSingleton { private final Map> providers = new ConcurrentHashMap<>(); - private Map notifiers = new ConcurrentHashMap<>(); + private ReactiveCacheContainer notifiers = ReactiveCacheContainer.create(); private NotifyConfigManager configManager; - private EventBus eventBus; + protected EventBus eventBus; - public DefaultNotifierManager(EventBus eventBus, NotifyConfigManager manager) { + private final ApplicationContext context; + + public DefaultNotifierManager(EventBus eventBus, NotifyConfigManager manager, + ApplicationContext context) { this.configManager = manager; this.eventBus = eventBus; + this.context = context; } protected Mono getProperties(NotifyType notifyType, @@ -44,7 +50,7 @@ public class DefaultNotifierManager implements NotifierManager, BeanPostProcesso } private Mono doReload(String id) { - log.debug("reload notifer config {}",id); + log.debug("reload notifer config {}", id); return Mono .justOrEmpty(notifiers.remove(id)) .flatMap(Notifier::close) @@ -59,20 +65,21 @@ public class DefaultNotifierManager implements NotifierManager, BeanPostProcesso .flatMap(map -> Mono.justOrEmpty(map.get(properties.getProvider()))) .switchIfEmpty(Mono.error(() -> new UnsupportedOperationException("不支持的服务商:" + properties.getProvider()))) .flatMap(notifierProvider -> notifierProvider.createNotifier(properties)) - //转成代理,把通知事件发送到消息网关中. - .map(notifier -> new NotifierEventDispatcher<>(eventBus, notifier)) - .flatMap(notifier -> Mono.justOrEmpty(notifiers.put(properties.getId(), notifier)) - .flatMap(Notifier::close)//如果存在旧的通知器则关掉之 - .thenReturn(notifier)); + .map(this::wrapNotifier); + } + + protected Notifier wrapNotifier(Notifier source) { + return new NotifierEventDispatcher<>(eventBus, source); } @Override @Nonnull public Mono getNotifier(@Nonnull NotifyType type, @Nonnull String id) { - return Mono - .justOrEmpty(notifiers.get(id)) - .switchIfEmpty(Mono.defer(() -> this.getProperties(type, id).flatMap(this::createNotifier))); + + return notifiers.computeIfAbsent(id, _id -> { + return this.getProperties(type, id).flatMap(this::createNotifier); + }); } public void registerProvider(NotifierProvider provider) { @@ -80,14 +87,6 @@ public class DefaultNotifierManager implements NotifierManager, BeanPostProcesso .put(provider.getProvider().getId(), provider); } - @Override - public Object postProcessAfterInitialization(Object bean, String beanName) throws BeansException { - if (bean instanceof NotifierProvider) { - registerProvider(((NotifierProvider) bean)); - } - return bean; - } - @Override public void run(String... args) throws Exception { @@ -109,4 +108,10 @@ public class DefaultNotifierManager implements NotifierManager, BeanPostProcesso .subscribe(); } + + @Override + public void afterSingletonsInstantiated() { + context.getBeanProvider(NotifierProvider.class) + .forEach(this::registerProvider); + } } diff --git a/jetlinks-components/notify-component/notify-core/src/main/java/org/jetlinks/community/notify/StaticTemplateManager.java b/jetlinks-components/notify-component/notify-core/src/main/java/org/jetlinks/community/notify/StaticTemplateManager.java index 9c1a5f69..083a2702 100644 --- a/jetlinks-components/notify-component/notify-core/src/main/java/org/jetlinks/community/notify/StaticTemplateManager.java +++ b/jetlinks-components/notify-component/notify-core/src/main/java/org/jetlinks/community/notify/StaticTemplateManager.java @@ -8,18 +8,24 @@ import org.jetlinks.community.notify.template.TemplateProperties; import org.jetlinks.community.notify.template.TemplateProvider; import org.jetlinks.core.event.EventBus; import org.springframework.beans.BeansException; +import org.springframework.beans.factory.SmartInitializingSingleton; import org.springframework.beans.factory.config.BeanPostProcessor; +import org.springframework.context.ApplicationContext; import reactor.core.publisher.Mono; @Getter @Setter -public class StaticTemplateManager extends AbstractTemplateManager implements BeanPostProcessor { +public class StaticTemplateManager extends AbstractTemplateManager implements SmartInitializingSingleton { private StaticNotifyProperties properties; - public StaticTemplateManager(StaticNotifyProperties properties, EventBus eventBus) { + private final ApplicationContext context; + + public StaticTemplateManager(StaticNotifyProperties properties, EventBus eventBus, + ApplicationContext context) { super(eventBus); this.properties = properties; + this.context = context; } public void register(TemplateProvider provider) { @@ -32,12 +38,8 @@ public class StaticTemplateManager extends AbstractTemplateManager implements Be } @Override - public Object postProcessAfterInitialization(Object bean, String beanName) throws BeansException { - - if(bean instanceof TemplateProvider){ - register(((TemplateProvider) bean)); - } - - return BeanPostProcessor.super.postProcessAfterInitialization(bean, beanName); + public void afterSingletonsInstantiated() { + context.getBeanProvider(TemplateProvider.class) + .forEach(this::register); } } diff --git a/jetlinks-components/notify-component/notify-core/src/main/java/org/jetlinks/community/notify/configuration/NotifierAutoConfiguration.java b/jetlinks-components/notify-component/notify-core/src/main/java/org/jetlinks/community/notify/configuration/NotifierAutoConfiguration.java index 8a4f22c5..ebb05a1f 100644 --- a/jetlinks-components/notify-component/notify-core/src/main/java/org/jetlinks/community/notify/configuration/NotifierAutoConfiguration.java +++ b/jetlinks-components/notify-component/notify-core/src/main/java/org/jetlinks/community/notify/configuration/NotifierAutoConfiguration.java @@ -2,18 +2,22 @@ package org.jetlinks.community.notify.configuration; import lombok.Generated; import org.jetlinks.community.notify.*; +import org.jetlinks.community.notify.template.TemplateProvider; import org.jetlinks.core.event.EventBus; import org.jetlinks.community.notify.*; import org.jetlinks.community.notify.template.TemplateManager; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.boot.autoconfigure.AutoConfiguration; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Primary; import java.util.List; -@Configuration(proxyBeanMethods = false) +@AutoConfiguration @EnableConfigurationProperties(StaticNotifyProperties.class) @Generated public class NotifierAutoConfiguration { @@ -29,10 +33,12 @@ public class NotifierAutoConfiguration { return new CompositeNotifyConfigManager(managers); } + @Bean public StaticTemplateManager staticTemplateManager(StaticNotifyProperties properties, - EventBus eventBus) { - return new StaticTemplateManager(properties, eventBus); + EventBus eventBus, + ApplicationContext context) { + return new StaticTemplateManager(properties, eventBus, context); } @Bean @@ -45,8 +51,10 @@ public class NotifierAutoConfiguration { @Bean @ConditionalOnMissingBean(NotifierManager.class) public DefaultNotifierManager notifierManager(EventBus eventBus, + ApplicationContext context, NotifyConfigManager configManager) { - return new DefaultNotifierManager(eventBus, configManager); + return new DefaultNotifierManager(eventBus, configManager, context); } + } diff --git a/jetlinks-components/notify-component/notify-core/src/main/java/org/jetlinks/community/notify/template/VariableDefinition.java b/jetlinks-components/notify-component/notify-core/src/main/java/org/jetlinks/community/notify/template/VariableDefinition.java index d79f7f25..d0cb55dd 100644 --- a/jetlinks-components/notify-component/notify-core/src/main/java/org/jetlinks/community/notify/template/VariableDefinition.java +++ b/jetlinks-components/notify-component/notify-core/src/main/java/org/jetlinks/community/notify/template/VariableDefinition.java @@ -15,7 +15,7 @@ import org.jetlinks.community.ConfigMetadataConstants; import org.jetlinks.community.relation.utils.VariableSource; import org.springframework.util.StringUtils; -import javax.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotBlank; import java.util.Date; import java.util.HashMap; import java.util.Map; diff --git a/jetlinks-components/notify-component/notify-core/src/main/resources/META-INF/spring.factories b/jetlinks-components/notify-component/notify-core/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports old mode 100755 new mode 100644 similarity index 60% rename from jetlinks-components/notify-component/notify-core/src/main/resources/META-INF/spring.factories rename to jetlinks-components/notify-component/notify-core/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports index 437b507e..2e151db5 --- a/jetlinks-components/notify-component/notify-core/src/main/resources/META-INF/spring.factories +++ b/jetlinks-components/notify-component/notify-core/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports @@ -1,3 +1,2 @@ -org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ -org.jetlinks.community.notify.configuration.NotifierAutoConfiguration,\ +org.jetlinks.community.notify.configuration.NotifierAutoConfiguration org.jetlinks.community.notify.configuration.RuleEngineNotifierConfiguration \ No newline at end of file diff --git a/jetlinks-components/notify-component/notify-core/src/main/resources/i18n/notify-component/messages_en.properties b/jetlinks-components/notify-component/notify-core/src/main/resources/i18n/notify-component/messages_en.properties new file mode 100644 index 00000000..0eab63ec --- /dev/null +++ b/jetlinks-components/notify-component/notify-core/src/main/resources/i18n/notify-component/messages_en.properties @@ -0,0 +1,27 @@ + +org.jetlinks.pro.notify.enums.DefaultVariable.recipient=recipient +org.jetlinks.pro.notify.enums.DefaultVariable.recipient_department=recipient department +org.jetlinks.pro.notify.enums.DefaultVariable.recipient_label=recipient label +org.jetlinks.pro.notify.enums.DefaultVariable.recipient_phone=recipient phone number +org.jetlinks.pro.notify.enums.DefaultVariable.called_number=called number +org.jetlinks.pro.notify.enums.DefaultVariable.content_link=content link +org.jetlinks.pro.notify.enums.DefaultVariable.picture_link=picture link +org.jetlinks.pro.notify.enums.DefaultVariable.template_jump_link=template jump link +org.jetlinks.pro.notify.enums.DefaultVariable.user_label=user label +org.jetlinks.pro.notify.enums.DefaultVariable.applet_appid=applet appid +org.jetlinks.pro.notify.enums.DefaultVariable.applet_jump_path=applet jump path + +org.jetlinks.pro.notify.enums.SubscriberTypeEnum.alarm=alarm +org.jetlinks.pro.notify.enums.SubscriberTypeEnum.systemEvent=systemEvent +org.jetlinks.pro.notify.enums.SubscriberTypeEnum.businessEvent=businessEvent +org.jetlinks.pro.notify.enums.SubscriberTypeEnum.other=other + +#message +message.subscriber.provider.system-event=system running abnormally + +message.notify.type.sms.name=SMS +message.notify.type.email.name=Email +message.notify.type.voice.name=Voice +message.notify.type.dingTalk.name=DingTalk +message.notify.type.weixin.name=WeChat +message.notify.type.webhook.name=WebHook \ No newline at end of file diff --git a/jetlinks-components/notify-component/notify-core/src/main/resources/i18n/notify-component/messages_zh.properties b/jetlinks-components/notify-component/notify-core/src/main/resources/i18n/notify-component/messages_zh.properties new file mode 100644 index 00000000..ef2511f9 --- /dev/null +++ b/jetlinks-components/notify-component/notify-core/src/main/resources/i18n/notify-component/messages_zh.properties @@ -0,0 +1,26 @@ +org.jetlinks.pro.notify.enums.DefaultVariable.recipient=\u6536\u4FE1\u4EBA +org.jetlinks.pro.notify.enums.DefaultVariable.recipient_department=\u6536\u4FE1\u4EBA\u90E8\u95E8 +org.jetlinks.pro.notify.enums.DefaultVariable.recipient_label=\u6536\u4FE1\u4EBA\u6807\u7B7E +org.jetlinks.pro.notify.enums.DefaultVariable.recipient_phone=\u6536\u4FE1\u4EBA\u624B\u673A\u53F7\u7801 +org.jetlinks.pro.notify.enums.DefaultVariable.called_number=\u88AB\u53EB\u53F7\u7801 +org.jetlinks.pro.notify.enums.DefaultVariable.content_link=\u5185\u5BB9\u94FE\u63A5 +org.jetlinks.pro.notify.enums.DefaultVariable.picture_link=\u56FE\u7247\u94FE\u63A5 +org.jetlinks.pro.notify.enums.DefaultVariable.template_jump_link=\u6A21\u677F\u8DF3\u8F6C\u94FE\u63A5 +org.jetlinks.pro.notify.enums.DefaultVariable.user_label=\u7528\u6237\u6807\u7B7E +org.jetlinks.pro.notify.enums.DefaultVariable.applet_appid=\u5C0F\u7A0B\u5E8Fappid +org.jetlinks.pro.notify.enums.DefaultVariable.applet_jump_path=\u5C0F\u7A0B\u5E8F\u8DF3\u8F6C\u8DEF\u5F84 + +org.jetlinks.pro.notify.enums.SubscriberTypeEnum.alarm=\u544A\u8B66 +org.jetlinks.pro.notify.enums.SubscriberTypeEnum.systemEvent=\u7CFB\u7EDF\u8FD0\u7EF4 +org.jetlinks.pro.notify.enums.SubscriberTypeEnum.businessEvent=\u4E1A\u52A1\u76D1\u63A7 +org.jetlinks.pro.notify.enums.SubscriberTypeEnum.other=\u5176\u5B83 + +#message +message.subscriber.provider.system-event=\u7CFB\u7EDF\u8FD0\u884C\u5F02\u5E38 + +message.notify.type.sms.name=\u77ED\u4FE1 +message.notify.type.email.name=\u90AE\u4EF6 +message.notify.type.voice.name=\u8BED\u8A00 +message.notify.type.dingTalk.name=\u9489\u9489 +message.notify.type.weixin.name=\u5FAE\u4FE1 +message.notify.type.webhook.name=WebHook \ No newline at end of file diff --git a/jetlinks-components/notify-component/notify-dingtalk/pom.xml b/jetlinks-components/notify-component/notify-dingtalk/pom.xml index 4868cd3b..ad32f8eb 100755 --- a/jetlinks-components/notify-component/notify-dingtalk/pom.xml +++ b/jetlinks-components/notify-component/notify-dingtalk/pom.xml @@ -5,7 +5,7 @@ notify-component org.jetlinks.community - 2.3.0-SNAPSHOT + 2.10.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/jetlinks-components/notify-component/notify-dingtalk/src/main/java/org/jetlinks/community/notify/dingtalk/corp/DingTalkMessageTemplate.java b/jetlinks-components/notify-component/notify-dingtalk/src/main/java/org/jetlinks/community/notify/dingtalk/corp/DingTalkMessageTemplate.java index f4e316ca..97753367 100755 --- a/jetlinks-components/notify-component/notify-dingtalk/src/main/java/org/jetlinks/community/notify/dingtalk/corp/DingTalkMessageTemplate.java +++ b/jetlinks-components/notify-component/notify-dingtalk/src/main/java/org/jetlinks/community/notify/dingtalk/corp/DingTalkMessageTemplate.java @@ -17,7 +17,7 @@ import org.springframework.web.util.UriComponentsBuilder; import reactor.core.publisher.Mono; import javax.annotation.Nonnull; -import javax.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotBlank; import java.util.ArrayList; import java.util.Collections; import java.util.List; diff --git a/jetlinks-components/notify-component/notify-dingtalk/src/main/java/org/jetlinks/community/notify/dingtalk/corp/DingTalkProperties.java b/jetlinks-components/notify-component/notify-dingtalk/src/main/java/org/jetlinks/community/notify/dingtalk/corp/DingTalkProperties.java index 057e7a75..d5634b6c 100755 --- a/jetlinks-components/notify-component/notify-dingtalk/src/main/java/org/jetlinks/community/notify/dingtalk/corp/DingTalkProperties.java +++ b/jetlinks-components/notify-component/notify-dingtalk/src/main/java/org/jetlinks/community/notify/dingtalk/corp/DingTalkProperties.java @@ -5,7 +5,7 @@ import lombok.Setter; import org.hswebframework.web.bean.FastBeanCopier; import org.jetlinks.community.notify.NotifierProperties; -import javax.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotBlank; @Getter @Setter diff --git a/jetlinks-components/notify-component/notify-email/pom.xml b/jetlinks-components/notify-component/notify-email/pom.xml index 0c07d5d7..e1dda95f 100755 --- a/jetlinks-components/notify-component/notify-email/pom.xml +++ b/jetlinks-components/notify-component/notify-email/pom.xml @@ -5,7 +5,7 @@ notify-component org.jetlinks.community - 2.3.0-SNAPSHOT + 2.10.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/jetlinks-components/notify-component/notify-email/src/main/java/org/jetlinks/community/notify/email/embedded/DefaultEmailNotifier.java b/jetlinks-components/notify-component/notify-email/src/main/java/org/jetlinks/community/notify/email/embedded/DefaultEmailNotifier.java index c3102f88..a937499c 100755 --- a/jetlinks-components/notify-component/notify-email/src/main/java/org/jetlinks/community/notify/email/embedded/DefaultEmailNotifier.java +++ b/jetlinks-components/notify-component/notify-email/src/main/java/org/jetlinks/community/notify/email/embedded/DefaultEmailNotifier.java @@ -2,7 +2,8 @@ package org.jetlinks.community.notify.email.embedded; import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBufUtil; -import io.netty.buffer.Unpooled; +import jakarta.mail.internet.MimeMessage; +import jakarta.mail.internet.MimeUtility; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; @@ -12,11 +13,13 @@ import org.hswebframework.web.bean.FastBeanCopier; import org.hswebframework.web.exception.BusinessException; import org.hswebframework.web.id.IDGenerator; import org.hswebframework.web.validator.ValidatorUtils; +import org.jetlinks.community.notify.AbstractNotifier; +import org.jetlinks.core.Values; import org.jetlinks.community.io.file.FileManager; import org.jetlinks.community.notify.*; import org.jetlinks.community.notify.email.EmailProvider; import org.jetlinks.community.notify.template.TemplateManager; -import org.jetlinks.core.Values; +import org.jetlinks.sdk.server.utils.ConverterUtils; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -25,7 +28,6 @@ import org.springframework.core.io.FileSystemResource; import org.springframework.core.io.InputStreamSource; import org.springframework.core.io.Resource; import org.springframework.core.io.buffer.DataBufferUtils; -import org.springframework.core.io.buffer.NettyDataBuffer; import org.springframework.http.MediaType; import org.springframework.mail.javamail.JavaMailSender; import org.springframework.mail.javamail.JavaMailSenderImpl; @@ -39,8 +41,6 @@ import reactor.core.scheduler.Scheduler; import reactor.core.scheduler.Schedulers; import javax.annotation.Nonnull; -import javax.mail.internet.MimeMessage; -import javax.mail.internet.MimeUtility; import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.LinkedHashMap; @@ -65,11 +65,11 @@ public class DefaultEmailNotifier extends AbstractNotifier { @Getter @Setter - private String sender; + private String username; @Getter @Setter - private String username; + private String sender; @Getter private final String notifierId; @@ -79,23 +79,27 @@ public class DefaultEmailNotifier extends AbstractNotifier { public static Scheduler scheduler = Schedulers.boundedElastic(); - private final FileManager fileManager; + private final WebClient webClient; + public DefaultEmailNotifier(NotifierProperties properties, TemplateManager templateManager, - FileManager fileManager) { + FileManager fileManager, + WebClient.Builder builder) { this(properties.getId(), - FastBeanCopier.copy(properties.getConfiguration(), new DefaultEmailProperties()), - templateManager, - fileManager); + FastBeanCopier.copy(properties.getConfiguration(), new DefaultEmailProperties()), + templateManager, + fileManager, + builder); } public DefaultEmailNotifier(String id, DefaultEmailProperties properties, TemplateManager templateManager, - FileManager fileManager) { + FileManager fileManager, + WebClient.Builder builder) { super(templateManager); ValidatorUtils.tryValidate(properties); JavaMailSenderImpl mailSender = new JavaMailSenderImpl(); @@ -106,17 +110,17 @@ public class DefaultEmailNotifier extends AbstractNotifier { mailSender.setJavaMailProperties(properties.createJavaMailProperties()); this.notifierId = id; this.sender = properties.getSender(); - this.username = properties.getUsername(); this.javaMailSender = mailSender; this.fileManager = fileManager; + this.webClient = builder.build(); } @Nonnull @Override public Mono send(@Nonnull EmailTemplate template, @Nonnull Values context) { return Mono.just(template) - .flatMap(temp -> convert(temp, context.getAllValues())) - .flatMap(this::doSend); + .flatMap(temp -> convert(temp, context.getAllValues())) + .flatMap(this::doSend); } @Nonnull @@ -192,8 +196,7 @@ public class DefaultEmailNotifier extends AbstractNotifier { protected Mono convertResource(String resource) { if (resource.startsWith("http")) { - return WebClient - .create() + return webClient .get() .uri(resource) .accept(MediaType.APPLICATION_OCTET_STREAM) @@ -213,9 +216,7 @@ public class DefaultEmailNotifier extends AbstractNotifier { .as(DataBufferUtils::join) .map(dataBuffer -> { try { - ByteBuf buf = dataBuffer instanceof NettyDataBuffer - ? ((NettyDataBuffer) dataBuffer).getNativeBuffer() - : Unpooled.wrappedBuffer(dataBuffer.asByteBuffer()); + ByteBuf buf = ConverterUtils.convertNettyBuffer(dataBuffer); return new ByteArrayResource(ByteBufUtil.getBytes(buf)); } finally { DataBufferUtils.release(dataBuffer); @@ -237,7 +238,7 @@ public class DefaultEmailNotifier extends AbstractNotifier { String subject = template.getSubject(); String text = template.getText(); if (CollectionUtils.isEmpty(sendToList) || ObjectUtils.isEmpty(subject) || ObjectUtils.isEmpty(text)) { - throw new BusinessException("模板内容错误，sendTo, text 或者 subject 不能为空."); + throw new BusinessException.NoStackTrace("模板内容错误，sendTo, text 或者 subject 不能为空."); } String sendText = template.render(text, context); @@ -299,5 +300,4 @@ public class DefaultEmailNotifier extends AbstractNotifier { return anyImage ? doc.html() : sendText; } - } diff --git a/jetlinks-components/notify-component/notify-email/src/main/java/org/jetlinks/community/notify/email/embedded/DefaultEmailNotifierProvider.java b/jetlinks-components/notify-component/notify-email/src/main/java/org/jetlinks/community/notify/email/embedded/DefaultEmailNotifierProvider.java index a5330c20..8e8e72b6 100755 --- a/jetlinks-components/notify-component/notify-email/src/main/java/org/jetlinks/community/notify/email/embedded/DefaultEmailNotifierProvider.java +++ b/jetlinks-components/notify-component/notify-email/src/main/java/org/jetlinks/community/notify/email/embedded/DefaultEmailNotifierProvider.java @@ -1,18 +1,16 @@ package org.jetlinks.community.notify.email.embedded; import org.hswebframework.web.i18n.LocaleUtils; -import org.jetlinks.community.notify.*; -import org.jetlinks.community.notify.email.EmailProvider; -import org.jetlinks.community.notify.template.TemplateProperties; -import org.jetlinks.community.notify.template.TemplateProvider; import org.jetlinks.core.metadata.ConfigMetadata; import org.jetlinks.core.metadata.DefaultConfigMetadata; import org.jetlinks.core.metadata.SimplePropertyMetadata; import org.jetlinks.core.metadata.types.*; import org.jetlinks.community.io.file.FileManager; import org.jetlinks.community.notify.*; +import org.jetlinks.community.notify.email.EmailProvider; import org.jetlinks.community.notify.template.TemplateManager; import org.springframework.stereotype.Component; +import org.springframework.web.reactive.function.client.WebClient; import reactor.core.publisher.Mono; import javax.annotation.Nonnull; @@ -20,17 +18,20 @@ import javax.annotation.Nonnull; import static org.jetlinks.community.ConfigMetadataConstants.*; @Component -public class DefaultEmailNotifierProvider implements NotifierProvider, TemplateProvider { +public class DefaultEmailNotifierProvider implements NotifierProvider { private final TemplateManager templateManager; - private final FileManager fileManager; + private final WebClient.Builder builder; + public DefaultEmailNotifierProvider(TemplateManager templateManager, - FileManager fileManager) { + FileManager fileManager, + WebClient.Builder builder) { this.templateManager = templateManager; this.fileManager = fileManager; + this.builder = builder; } @Nonnull @@ -101,8 +102,6 @@ public class DefaultEmailNotifierProvider implements NotifierProvider, TemplateP .addPropertyMetadata(value) .addPropertyMetadata(description))); } - - } @Override @@ -110,22 +109,11 @@ public class DefaultEmailNotifierProvider implements NotifierProvider, TemplateP return notifierConfig; } - @Override - public ConfigMetadata getTemplateConfigMetadata() { - return templateConfig; - } - @Nonnull @Override public Mono createNotifier(@Nonnull NotifierProperties properties) { - return Mono.fromSupplier(() -> new DefaultEmailNotifier(properties, templateManager,fileManager)) + return Mono.fromSupplier(() -> new DefaultEmailNotifier(properties, templateManager, fileManager, builder)) .as(LocaleUtils::transform); } - @Override - public Mono createTemplate(TemplateProperties properties) { - - return Mono.fromSupplier(() -> new EmailTemplate().with(properties).validate()) - .as(LocaleUtils::transform); - } } diff --git a/jetlinks-components/notify-component/notify-email/src/main/java/org/jetlinks/community/notify/email/embedded/DefaultEmailProperties.java b/jetlinks-components/notify-component/notify-email/src/main/java/org/jetlinks/community/notify/email/embedded/DefaultEmailProperties.java index b23c6391..54f2d7f7 100755 --- a/jetlinks-components/notify-component/notify-email/src/main/java/org/jetlinks/community/notify/email/embedded/DefaultEmailProperties.java +++ b/jetlinks-components/notify-component/notify-email/src/main/java/org/jetlinks/community/notify/email/embedded/DefaultEmailProperties.java @@ -6,7 +6,6 @@ import lombok.NoArgsConstructor; import lombok.Setter; import java.util.List; -import java.util.Map; import java.util.Properties; @Getter diff --git a/jetlinks-components/notify-component/notify-email/src/main/java/org/jetlinks/community/notify/email/embedded/DefaultEmailTemplateProvider.java b/jetlinks-components/notify-component/notify-email/src/main/java/org/jetlinks/community/notify/email/embedded/DefaultEmailTemplateProvider.java new file mode 100755 index 00000000..0fe27aa2 --- /dev/null +++ b/jetlinks-components/notify-component/notify-email/src/main/java/org/jetlinks/community/notify/email/embedded/DefaultEmailTemplateProvider.java @@ -0,0 +1,110 @@ +package org.jetlinks.community.notify.email.embedded; + +import org.hswebframework.web.i18n.LocaleUtils; +import org.jetlinks.core.metadata.ConfigMetadata; +import org.jetlinks.core.metadata.DefaultConfigMetadata; +import org.jetlinks.core.metadata.SimplePropertyMetadata; +import org.jetlinks.core.metadata.types.*; +import org.jetlinks.community.notify.*; +import org.jetlinks.community.notify.email.EmailProvider; +import org.jetlinks.community.notify.template.TemplateProperties; +import org.jetlinks.community.notify.template.TemplateProvider; +import org.springframework.stereotype.Component; +import reactor.core.publisher.Mono; + +import javax.annotation.Nonnull; + +import static org.jetlinks.community.ConfigMetadataConstants.*; + +@Component +public class DefaultEmailTemplateProvider implements TemplateProvider { + + public DefaultEmailTemplateProvider() { + + } + + @Nonnull + @Override + public NotifyType getType() { + return DefaultNotifyType.email; + } + + @Nonnull + @Override + public Provider getProvider() { + return EmailProvider.embedded; + } + + public static final DefaultConfigMetadata templateConfig; + + public static final DefaultConfigMetadata notifierConfig; + + static { + { + SimplePropertyMetadata name = new SimplePropertyMetadata(); + name.setId("name"); + name.setName("文件名"); + name.setValueType(new StringType()); + + SimplePropertyMetadata location = new SimplePropertyMetadata(); + location.setId("location"); + location.setName("文件地址"); + location.setValueType(new FileType() + .bodyType(FileType.BodyType.url) + .expand(allowInput.value(true))); + + templateConfig = new DefaultConfigMetadata("邮件模版", "") + .add("subject", "标题", "标题,可使用变量", new StringType().expand(maxLength.value(255L))) + .add("text", "内容", "", new StringType().expand(maxLength.value(5120L), isRichText.value(true))) + .add("sendTo", "收件人", "", new ArrayType().elementType(new StringType())) + .add("attachments", "附件列表", "", new ArrayType() + .elementType(new ObjectType() + .addPropertyMetadata(name) + .addPropertyMetadata(location))); + } + + { + SimplePropertyMetadata name = new SimplePropertyMetadata(); + name.setId("name"); + name.setName("配置名称"); + name.setValueType(new StringType()); + + SimplePropertyMetadata value = new SimplePropertyMetadata(); + value.setId("value"); + value.setName("配置值"); + value.setValueType(new StringType()); + + SimplePropertyMetadata description = new SimplePropertyMetadata(); + description.setId("description"); + description.setName("说明"); + description.setValueType(new StringType()); + + notifierConfig = new DefaultConfigMetadata("邮件配置", "") + .add("host", "服务器地址", "例如: pop3.qq.com", new StringType().expand(maxLength.value(255L))) + .add("port", "端口", "", new IntType().min(0).max(65536)) + .add("sender", "发件人", "默认和用户名相同", new StringType()) + .add("username", "用户名", "", new StringType()) + .add("password", "密码", "", new PasswordType()) + .add("properties", "其他配置", "", new ArrayType() + .elementType(new ObjectType() + .addPropertyMetadata(name) + .addPropertyMetadata(value) + .addPropertyMetadata(description))); + } + + + } + + @Override + public ConfigMetadata getTemplateConfigMetadata() { + return templateConfig; + } + + + @Override + public Mono createTemplate(TemplateProperties properties) { + + return Mono.fromSupplier(() -> new EmailTemplate().with(properties).validate()) + .as(LocaleUtils::transform); + } +} diff --git a/jetlinks-components/notify-component/notify-email/src/main/java/org/jetlinks/community/notify/email/embedded/EmailTemplate.java b/jetlinks-components/notify-component/notify-email/src/main/java/org/jetlinks/community/notify/email/embedded/EmailTemplate.java index 1cc45018..a497f48a 100755 --- a/jetlinks-components/notify-component/notify-email/src/main/java/org/jetlinks/community/notify/email/embedded/EmailTemplate.java +++ b/jetlinks-components/notify-component/notify-email/src/main/java/org/jetlinks/community/notify/email/embedded/EmailTemplate.java @@ -3,27 +3,23 @@ package org.jetlinks.community.notify.email.embedded; import lombok.*; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; -import org.jetlinks.community.notify.NotifyVariableBusinessConstant; +import org.hswebframework.web.i18n.LocaleUtils; import org.jetlinks.community.notify.template.AbstractTemplate; import org.jetlinks.community.notify.template.VariableDefinition; +import org.jetlinks.community.relation.RelationConstants; +import org.jetlinks.community.relation.utils.VariableSource; import org.jetlinks.community.utils.ConverterUtils; import org.jetlinks.core.metadata.types.ArrayType; import org.jetlinks.core.metadata.types.FileType; +import org.jetlinks.community.notify.NotifyVariableBusinessConstant; import org.jetlinks.community.notify.template.Variable; -import org.jetlinks.community.relation.RelationConstants; -import org.jetlinks.community.relation.utils.VariableSource; import reactor.core.publisher.Mono; import javax.annotation.Nonnull; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.function.Function; -import java.util.stream.Collectors; - -import static org.jetlinks.community.notify.email.embedded.EmailTemplate.Attachment.LOCATION_KEY; -import static org.jetlinks.community.notify.email.embedded.EmailTemplate.Attachment.locationKey; @Getter @Setter @@ -81,7 +77,7 @@ public class EmailTemplate extends AbstractTemplate { .id(SEND_TO_KEY) .name("收件人") .expand(NotifyVariableBusinessConstant.businessId, - NotifyVariableBusinessConstant.NotifyVariableBusinessTypes.userType) + NotifyVariableBusinessConstant.NotifyVariableBusinessTypes.userType) .required(true) .type(ArrayType.ID) .build() @@ -98,7 +94,7 @@ public class EmailTemplate extends AbstractTemplate { variables.add( VariableDefinition .builder() - .id(locationKey(index)) + .id(Attachment.locationKey(index)) .name(attachment.getName()) .type(FileType.ID) .description(attachment.getName()) diff --git a/jetlinks-components/notify-component/notify-sms/pom.xml b/jetlinks-components/notify-component/notify-sms/pom.xml index 70b3d8df..9e9b99c7 100755 --- a/jetlinks-components/notify-component/notify-sms/pom.xml +++ b/jetlinks-components/notify-component/notify-sms/pom.xml @@ -5,7 +5,7 @@ notify-component org.jetlinks.community - 2.3.0-SNAPSHOT + 2.10.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/jetlinks-components/notify-component/notify-sms/src/main/java/org/jetlinks/community/notify/sms/aliyun/AliyunSmsTemplate.java b/jetlinks-components/notify-component/notify-sms/src/main/java/org/jetlinks/community/notify/sms/aliyun/AliyunSmsTemplate.java index 254622aa..b7a37ece 100755 --- a/jetlinks-components/notify-component/notify-sms/src/main/java/org/jetlinks/community/notify/sms/aliyun/AliyunSmsTemplate.java +++ b/jetlinks-components/notify-component/notify-sms/src/main/java/org/jetlinks/community/notify/sms/aliyun/AliyunSmsTemplate.java @@ -14,7 +14,7 @@ import org.springframework.util.StringUtils; import reactor.core.publisher.Flux; import javax.annotation.Nonnull; -import javax.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotBlank; import java.util.Collections; import java.util.List; import java.util.Map; diff --git a/jetlinks-components/notify-component/notify-voice/pom.xml b/jetlinks-components/notify-component/notify-voice/pom.xml index 5a9bb7da..9a64c653 100644 --- a/jetlinks-components/notify-component/notify-voice/pom.xml +++ b/jetlinks-components/notify-component/notify-voice/pom.xml @@ -5,7 +5,7 @@ notify-component org.jetlinks.community - 2.3.0-SNAPSHOT + 2.10.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/jetlinks-components/notify-component/notify-voice/src/main/java/org/jetlinks/community/notify/voice/aliyun/AliyunVoiceTemplate.java b/jetlinks-components/notify-component/notify-voice/src/main/java/org/jetlinks/community/notify/voice/aliyun/AliyunVoiceTemplate.java index c4084d24..45dc6ca9 100755 --- a/jetlinks-components/notify-component/notify-voice/src/main/java/org/jetlinks/community/notify/voice/aliyun/AliyunVoiceTemplate.java +++ b/jetlinks-components/notify-component/notify-voice/src/main/java/org/jetlinks/community/notify/voice/aliyun/AliyunVoiceTemplate.java @@ -14,7 +14,7 @@ import org.springframework.util.StringUtils; import reactor.core.publisher.Flux; import javax.annotation.Nonnull; -import javax.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotBlank; import java.util.Collections; import java.util.List; import java.util.Map; diff --git a/jetlinks-components/notify-component/notify-webhook/pom.xml b/jetlinks-components/notify-component/notify-webhook/pom.xml index db4f4e98..60c77bba 100644 --- a/jetlinks-components/notify-component/notify-webhook/pom.xml +++ b/jetlinks-components/notify-component/notify-webhook/pom.xml @@ -5,7 +5,7 @@ notify-component org.jetlinks.community - 2.3.0-SNAPSHOT + 2.10.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/jetlinks-components/notify-component/notify-webhook/src/main/java/org/jetlinks/community/notify/webhook/http/HttpWebHookProperties.java b/jetlinks-components/notify-component/notify-webhook/src/main/java/org/jetlinks/community/notify/webhook/http/HttpWebHookProperties.java index 3ff612fd..f612d939 100644 --- a/jetlinks-components/notify-component/notify-webhook/src/main/java/org/jetlinks/community/notify/webhook/http/HttpWebHookProperties.java +++ b/jetlinks-components/notify-component/notify-webhook/src/main/java/org/jetlinks/community/notify/webhook/http/HttpWebHookProperties.java @@ -5,7 +5,7 @@ import lombok.Getter; import lombok.Setter; import org.hibernate.validator.constraints.URL; -import javax.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotBlank; import java.util.List; @Getter diff --git a/jetlinks-components/notify-component/notify-wechat/pom.xml b/jetlinks-components/notify-component/notify-wechat/pom.xml index cf76944c..7e18d9e6 100755 --- a/jetlinks-components/notify-component/notify-wechat/pom.xml +++ b/jetlinks-components/notify-component/notify-wechat/pom.xml @@ -5,7 +5,7 @@ notify-component org.jetlinks.community - 2.3.0-SNAPSHOT + 2.10.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/jetlinks-components/notify-component/notify-wechat/src/main/java/org/jetlinks/community/notify/wechat/corp/WechatCorpProperties.java b/jetlinks-components/notify-component/notify-wechat/src/main/java/org/jetlinks/community/notify/wechat/corp/WechatCorpProperties.java index 07c35971..861b9104 100644 --- a/jetlinks-components/notify-component/notify-wechat/src/main/java/org/jetlinks/community/notify/wechat/corp/WechatCorpProperties.java +++ b/jetlinks-components/notify-component/notify-wechat/src/main/java/org/jetlinks/community/notify/wechat/corp/WechatCorpProperties.java @@ -5,7 +5,7 @@ import lombok.Setter; import org.hswebframework.web.bean.FastBeanCopier; import org.jetlinks.community.notify.NotifierProperties; -import javax.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotBlank; @Getter @Setter diff --git a/jetlinks-components/notify-component/notify-wechat/src/main/java/org/jetlinks/community/notify/wechat/corp/WechatMessageTemplate.java b/jetlinks-components/notify-component/notify-wechat/src/main/java/org/jetlinks/community/notify/wechat/corp/WechatMessageTemplate.java index 67dfda64..0190e20b 100644 --- a/jetlinks-components/notify-component/notify-wechat/src/main/java/org/jetlinks/community/notify/wechat/corp/WechatMessageTemplate.java +++ b/jetlinks-components/notify-component/notify-wechat/src/main/java/org/jetlinks/community/notify/wechat/corp/WechatMessageTemplate.java @@ -17,7 +17,7 @@ import org.springframework.web.util.UriComponentsBuilder; import reactor.core.publisher.Mono; import javax.annotation.Nonnull; -import javax.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotBlank; import java.util.ArrayList; import java.util.Collections; import java.util.List; diff --git a/jetlinks-components/notify-component/pom.xml b/jetlinks-components/notify-component/pom.xml index 57cc1e87..2c70aec9 100644 --- a/jetlinks-components/notify-component/pom.xml +++ b/jetlinks-components/notify-component/pom.xml @@ -5,7 +5,7 @@ jetlinks-components org.jetlinks.community - 2.3.0-SNAPSHOT + 2.10.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/jetlinks-components/pom.xml b/jetlinks-components/pom.xml index b6327805..8578276f 100644 --- a/jetlinks-components/pom.xml +++ b/jetlinks-components/pom.xml @@ -5,7 +5,7 @@ jetlinks-community org.jetlinks.community - 2.3.0-SNAPSHOT + 2.10.0-SNAPSHOT ../pom.xml 4.0.0 @@ -28,6 +28,8 @@ protocol-component relation-component tdengine-component + timescaledb-component + datasource-component jetlinks-components diff --git a/jetlinks-components/protocol-component/pom.xml b/jetlinks-components/protocol-component/pom.xml index aece1ebd..e6abd9ee 100755 --- a/jetlinks-components/protocol-component/pom.xml +++ b/jetlinks-components/protocol-component/pom.xml @@ -5,7 +5,7 @@ jetlinks-components org.jetlinks.community - 2.3.0-SNAPSHOT + 2.10.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/jetlinks-components/protocol-component/src/main/java/org/jetlinks/community/protocol/ProtocolSupportEntity.java b/jetlinks-components/protocol-component/src/main/java/org/jetlinks/community/protocol/ProtocolSupportEntity.java index 66351f6c..f151157e 100644 --- a/jetlinks-components/protocol-component/src/main/java/org/jetlinks/community/protocol/ProtocolSupportEntity.java +++ b/jetlinks-components/protocol-component/src/main/java/org/jetlinks/community/protocol/ProtocolSupportEntity.java @@ -23,7 +23,7 @@ import org.jetlinks.supports.protocol.management.ProtocolSupportDefinition; import javax.persistence.Column; import javax.persistence.GeneratedValue; import javax.persistence.Table; -import javax.validation.constraints.Pattern; +import jakarta.validation.constraints.Pattern; import java.sql.JDBCType; import java.util.Arrays; import java.util.List; diff --git a/jetlinks-components/protocol-component/src/main/java/org/jetlinks/community/protocol/configuration/LazyProtocolSupports.java b/jetlinks-components/protocol-component/src/main/java/org/jetlinks/community/protocol/configuration/LazyProtocolSupports.java index 6cdd51aa..ecc9bfde 100644 --- a/jetlinks-components/protocol-component/src/main/java/org/jetlinks/community/protocol/configuration/LazyProtocolSupports.java +++ b/jetlinks-components/protocol-component/src/main/java/org/jetlinks/community/protocol/configuration/LazyProtocolSupports.java @@ -3,16 +3,33 @@ package org.jetlinks.community.protocol.configuration; import org.jetlinks.core.ProtocolSupports; import org.jetlinks.core.defaults.CompositeProtocolSupports; import org.springframework.beans.BeansException; +import org.springframework.beans.factory.SmartInitializingSingleton; import org.springframework.beans.factory.config.BeanPostProcessor; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; + +public class LazyProtocolSupports extends CompositeProtocolSupports + implements SmartInitializingSingleton, ApplicationContextAware { + + + private ApplicationContext applicationContext; -public class LazyProtocolSupports extends CompositeProtocolSupports implements BeanPostProcessor { @Override - public Object postProcessAfterInitialization(Object bean, String beanName) throws BeansException { - - if(bean instanceof ProtocolSupports){ - register(((ProtocolSupports) bean)); + public void afterSingletonsInstantiated() { + for (ProtocolSupports value : applicationContext + .getBeansOfType(ProtocolSupports.class) + .values()) { + if (value == this) { + continue; + } + register(value); } - return BeanPostProcessor.super.postProcessAfterInitialization(bean, beanName); + } + + @Override + public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { + this.applicationContext = applicationContext; + } } diff --git a/jetlinks-components/protocol-component/src/main/java/org/jetlinks/community/protocol/configuration/ProtocolAutoConfiguration.java b/jetlinks-components/protocol-component/src/main/java/org/jetlinks/community/protocol/configuration/ProtocolAutoConfiguration.java index ae3cdcf6..f2f4cd95 100644 --- a/jetlinks-components/protocol-component/src/main/java/org/jetlinks/community/protocol/configuration/ProtocolAutoConfiguration.java +++ b/jetlinks-components/protocol-component/src/main/java/org/jetlinks/community/protocol/configuration/ProtocolAutoConfiguration.java @@ -10,12 +10,9 @@ import org.jetlinks.core.event.EventBus; import org.jetlinks.core.spi.ServiceContext; import org.jetlinks.community.configure.device.DeviceClusterConfiguration; import org.jetlinks.community.io.file.FileManager; -import org.jetlinks.community.protocol.*; import org.jetlinks.supports.protocol.StaticProtocolSupports; -import org.jetlinks.supports.protocol.management.ClusterProtocolSupportManager; import org.jetlinks.supports.protocol.management.ProtocolSupportLoader; import org.jetlinks.supports.protocol.management.ProtocolSupportLoaderProvider; -import org.jetlinks.supports.protocol.management.ProtocolSupportManager; import org.springframework.beans.factory.ObjectProvider; import org.springframework.boot.autoconfigure.AutoConfigureBefore; import org.springframework.context.ApplicationContext; diff --git a/jetlinks-components/relation-component/pom.xml b/jetlinks-components/relation-component/pom.xml index 79fb6199..6470ab13 100644 --- a/jetlinks-components/relation-component/pom.xml +++ b/jetlinks-components/relation-component/pom.xml @@ -5,7 +5,7 @@ jetlinks-components org.jetlinks.community - 2.3.0-SNAPSHOT + 2.10.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/jetlinks-components/relation-component/src/main/java/org/jetlinks/community/relation/entity/RelatedEntity.java b/jetlinks-components/relation-component/src/main/java/org/jetlinks/community/relation/entity/RelatedEntity.java index 249bd5f4..4fa662f4 100644 --- a/jetlinks-components/relation-component/src/main/java/org/jetlinks/community/relation/entity/RelatedEntity.java +++ b/jetlinks-components/relation-component/src/main/java/org/jetlinks/community/relation/entity/RelatedEntity.java @@ -15,7 +15,7 @@ import org.springframework.util.StringUtils; import javax.persistence.Column; import javax.persistence.Index; import javax.persistence.Table; -import javax.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotBlank; import java.util.Collection; import java.util.Set; import java.util.stream.Collectors; diff --git a/jetlinks-components/rule-engine-component/pom.xml b/jetlinks-components/rule-engine-component/pom.xml index e1166d55..d945295a 100644 --- a/jetlinks-components/rule-engine-component/pom.xml +++ b/jetlinks-components/rule-engine-component/pom.xml @@ -5,7 +5,7 @@ jetlinks-components org.jetlinks.community - 2.3.0-SNAPSHOT + 2.10.0-SNAPSHOT ../pom.xml 4.0.0 @@ -13,11 +13,7 @@ rule-engine-component - - com.cronutils - cron-utils - 9.1.6 - + org.jetlinks rule-engine-support @@ -32,7 +28,7 @@ ${project.groupId} - elasticsearch-component + timeseries-component ${project.version} @@ -69,6 +65,16 @@ relation-component ${project.version} + + org.jetlinks.community + configure-component + 2.10.0-SNAPSHOT + compile + + + org.jetlinks.community + things-component + \ No newline at end of file diff --git a/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/cluster/ClusterSchedulerLoadBalancer.java b/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/cluster/ClusterSchedulerLoadBalancer.java new file mode 100755 index 00000000..67f4b3d7 --- /dev/null +++ b/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/cluster/ClusterSchedulerLoadBalancer.java @@ -0,0 +1,66 @@ +package org.jetlinks.community.rule.engine.cluster; + +import lombok.Generated; +import lombok.extern.slf4j.Slf4j; +import org.jetlinks.core.event.EventBus; +import org.jetlinks.core.trace.MonoTracer; +import org.jetlinks.rule.engine.api.scheduler.SchedulerSelector; +import org.jetlinks.rule.engine.cluster.RuleInstanceRepository; +import org.jetlinks.rule.engine.cluster.SchedulerRegistry; +import org.jetlinks.rule.engine.cluster.TaskSnapshotRepository; +import org.jetlinks.rule.engine.cluster.balancer.DefaultSchedulerLoadBalancer; +import org.springframework.boot.CommandLineRunner; +import reactor.core.publisher.Mono; + +import javax.annotation.PreDestroy; +import java.time.Duration; + +/** + * 集群调度负载均衡器，用于对调度器任务进行负载均衡处理 + * + * @author zhouhao + * @since 1.3 + */ +@Slf4j +@Generated +public class ClusterSchedulerLoadBalancer extends DefaultSchedulerLoadBalancer implements CommandLineRunner { + + private final RuleInstanceRepository instanceRepository; + + public ClusterSchedulerLoadBalancer(EventBus eventBus, + SchedulerRegistry registry, + TaskSnapshotRepository snapshotRepository, + RuleInstanceRepository instanceRepository, + SchedulerSelector selector) { + super(eventBus, registry, snapshotRepository, selector); + this.instanceRepository = instanceRepository; + } + + @Override + @PreDestroy + public void cleanup() { + super.cleanup(); + } + + @Override + public void run(String... args) { + + this + .setupAsync()//恢复之前的调度 + .delayElement(Duration.ofMillis(2000)) + .then( + instanceRepository + .findAll() + .flatMap(instance -> this + //使用本地调度器进行负载,实现集群新增节点时的弹性调度 + .reBalance(registry.getLocalSchedulers(), instance, true) + .onErrorResume(err -> { + log.error("Re Balance Rule [{}] error", instance.getId(), err); + return Mono.empty(); + })) + .then() + ) + .as(MonoTracer.create("/rule-engine/scheduler/startup")) + .subscribe(); + } +} diff --git a/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/configuration/RuleEngineConfiguration.java b/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/configuration/RuleEngineConfiguration.java index 44444808..40f7cc61 100644 --- a/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/configuration/RuleEngineConfiguration.java +++ b/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/configuration/RuleEngineConfiguration.java @@ -1,36 +1,58 @@ package org.jetlinks.community.rule.engine.configuration; import lombok.extern.slf4j.Slf4j; +import org.hswebframework.ezorm.rdb.mapping.ReactiveRepository; +import org.jetlinks.community.configure.device.DeviceClusterConfiguration; +import org.jetlinks.community.rule.engine.cluster.ClusterSchedulerLoadBalancer; +import org.jetlinks.community.rule.engine.commons.ShakeLimitProvider; import org.jetlinks.community.rule.engine.commons.TermsConditionEvaluator; +import org.jetlinks.community.rule.engine.entity.TaskSnapshotEntity; import org.jetlinks.community.rule.engine.executor.DeviceSelectorBuilder; import org.jetlinks.community.rule.engine.executor.device.DeviceDataTaskExecutorProvider; +import org.jetlinks.community.rule.engine.io.EventBusRuleIOManager; +import org.jetlinks.community.rule.engine.log.TimeSeriesRuleEngineLogService; +import org.jetlinks.community.rule.engine.repository.LocalTaskSnapshotRepository; +import org.jetlinks.community.things.configuration.ThingsConfiguration; +import org.jetlinks.community.timeseries.TimeSeriesManager; +import org.jetlinks.core.cluster.ClusterManager; import org.jetlinks.core.event.EventBus; +import org.jetlinks.core.rpc.RpcManager; import org.jetlinks.core.things.ThingsDataManager; +import org.jetlinks.rule.engine.api.RuleData; import org.jetlinks.rule.engine.api.RuleEngine; import org.jetlinks.rule.engine.api.scheduler.Scheduler; +import org.jetlinks.rule.engine.api.scheduler.SchedulerSelector; import org.jetlinks.rule.engine.api.task.ConditionEvaluator; -import org.jetlinks.rule.engine.api.task.TaskExecutorProvider; import org.jetlinks.rule.engine.api.worker.Worker; -import org.jetlinks.rule.engine.condition.ConditionEvaluatorStrategy; +import org.jetlinks.rule.engine.cluster.ClusterRuleEngine; +import org.jetlinks.rule.engine.cluster.RuleInstanceRepository; +import org.jetlinks.rule.engine.cluster.SchedulerRegistry; +import org.jetlinks.rule.engine.cluster.TaskSnapshotRepository; +import org.jetlinks.rule.engine.cluster.scheduler.ClusterRpcSchedulerRegistry; import org.jetlinks.rule.engine.condition.DefaultConditionEvaluator; import org.jetlinks.rule.engine.condition.supports.DefaultScriptEvaluator; import org.jetlinks.rule.engine.condition.supports.ScriptConditionEvaluatorStrategy; import org.jetlinks.rule.engine.condition.supports.ScriptEvaluator; -import org.jetlinks.rule.engine.defaults.DefaultRuleEngine; import org.jetlinks.rule.engine.defaults.LocalScheduler; -import org.jetlinks.rule.engine.defaults.LocalWorker; import org.jetlinks.rule.engine.model.DefaultRuleModelParser; -import org.jetlinks.rule.engine.model.RuleModelParserStrategy; import org.jetlinks.rule.engine.model.antv.AntVG6RuleModelParserStrategy; -import org.springframework.beans.BeansException; -import org.springframework.beans.factory.config.BeanPostProcessor; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.beans.factory.SmartInitializingSingleton; +import org.springframework.boot.autoconfigure.AutoConfiguration; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -@Configuration +@AutoConfiguration(after = {DeviceClusterConfiguration.class, ThingsConfiguration.class}) @Slf4j +@EnableConfigurationProperties(RuleEngineProperties.class) public class RuleEngineConfiguration { + static { + RuleData.create("load-serializer"); + } + @Bean public DefaultRuleModelParser defaultRuleModelParser() { return new DefaultRuleModelParser(); @@ -42,48 +64,56 @@ public class RuleEngineConfiguration { } @Bean - public TermsConditionEvaluator termsConditionEvaluator(){ + public TermsConditionEvaluator termsConditionEvaluator() { return new TermsConditionEvaluator(); } - @Bean - public AntVG6RuleModelParserStrategy antVG6RuleModelParserStrategy() { - return new AntVG6RuleModelParserStrategy(); - } - - @Bean - public Scheduler localScheduler(Worker worker) { - LocalScheduler scheduler = new LocalScheduler("local"); - scheduler.addWorker(worker); + @Bean(destroyMethod = "dispose") + public LocalScheduler ruleScheduler(RuleEngineProperties properties, + ObjectProvider workers) { + LocalScheduler scheduler = new LocalScheduler(properties.getServerId()); + workers.forEach(scheduler::addWorker); return scheduler; } + @Bean - public BeanPostProcessor autoRegisterStrategy(DefaultRuleModelParser defaultRuleModelParser, - DefaultConditionEvaluator defaultConditionEvaluator, - LocalWorker worker) { - return new BeanPostProcessor() { - @Override - public Object postProcessBeforeInitialization(Object bean, String beanName) throws BeansException { + public SchedulerSelector schedulerSelector() { + return SchedulerSelector.selectAll; + } - return bean; - } - @Override - public Object postProcessAfterInitialization(Object bean, String beanName) throws BeansException { - if (bean instanceof RuleModelParserStrategy) { - defaultRuleModelParser.register(((RuleModelParserStrategy) bean)); - } - if (bean instanceof ConditionEvaluatorStrategy) { - defaultConditionEvaluator.register(((ConditionEvaluatorStrategy) bean)); - } - if (bean instanceof TaskExecutorProvider) { - worker.addExecutor(((TaskExecutorProvider) bean)); - } + @Bean + public SchedulerRegistry schedulerRegistry(RpcManager rpcManager, + ObjectProvider schedulers, + RuleEngineProperties properties) { + ClusterRpcSchedulerRegistry registry = new ClusterRpcSchedulerRegistry(properties.getNamespace(), rpcManager); - return bean; - } - }; + schedulers.forEach(registry::register); + + return registry; + } + + + @Bean + public TaskSnapshotRepository taskSnapshotRepository(ReactiveRepository repository) { + return new LocalTaskSnapshotRepository(repository); + } + + @Bean + @ConditionalOnBean({ + RuleInstanceRepository.class, + SchedulerRegistry.class + }) + public ClusterSchedulerLoadBalancer clusterSchedulerLoadBalancer(EventBus eventBus, + SchedulerRegistry registry, + TaskSnapshotRepository taskSnapshotRepository, + RuleInstanceRepository instanceRepository, + SchedulerSelector schedulerSelector) { + return new ClusterSchedulerLoadBalancer(eventBus, registry, + taskSnapshotRepository, + instanceRepository, + schedulerSelector); } @Bean @@ -97,14 +127,29 @@ public class RuleEngineConfiguration { } @Bean - public LocalWorker localWorker(EventBus eventBus, ConditionEvaluator evaluator) { - return new LocalWorker("local", "local", eventBus, evaluator); + public SpringClusterWorker clusterWorker(RuleEngineProperties properties, + EventBus eventBus, + ClusterManager clusterManager, + ConditionEvaluator evaluator, + ApplicationContext context) { + return new SpringClusterWorker(properties.getServerId(), + properties.getServerName(), + eventBus, + new EventBusRuleIOManager(eventBus, evaluator, clusterManager), + context); + } - @Bean - public RuleEngine defaultRuleEngine(Scheduler scheduler) { - return new DefaultRuleEngine(scheduler); + @ConditionalOnBean({ + SchedulerRegistry.class, + SchedulerSelector.class, + TaskSnapshotRepository.class, + }) + public RuleEngine ruleEngine(SchedulerRegistry registry, + TaskSnapshotRepository repository, + SchedulerSelector selector) { + return new ClusterRuleEngine(registry, repository, selector); } @Bean @@ -113,4 +158,17 @@ public class RuleEngineConfiguration { return new DeviceDataTaskExecutorProvider(dataManager, selectorBuilder); } + @Bean + public TimeSeriesRuleEngineLogService ruleEngineLogService(TimeSeriesManager timeSeriesManager) { + return new TimeSeriesRuleEngineLogService(timeSeriesManager); + } + + @Bean + public SmartInitializingSingleton shakeLimitProviderRegister(ApplicationContext context) { + return () -> context + .getBeansOfType(ShakeLimitProvider.class) + .values() + .forEach(provider -> ShakeLimitProvider.supports.register(provider.provider(), provider)); + } + } diff --git a/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/configuration/RuleEngineLogIndexInitialize.java b/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/configuration/RuleEngineLogIndexInitialize.java deleted file mode 100644 index bdb6796a..00000000 --- a/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/configuration/RuleEngineLogIndexInitialize.java +++ /dev/null @@ -1,39 +0,0 @@ -package org.jetlinks.community.rule.engine.configuration; - -import lombok.extern.slf4j.Slf4j; -import org.jetlinks.community.elastic.search.index.DefaultElasticSearchIndexMetadata; -import org.jetlinks.community.elastic.search.index.ElasticSearchIndexManager; -import org.jetlinks.community.rule.engine.event.handler.RuleEngineLoggerIndexProvider; -import org.jetlinks.core.metadata.types.DateTimeType; -import org.jetlinks.core.metadata.types.StringType; -import org.springframework.core.annotation.Order; -import org.springframework.stereotype.Component; - -/** - * @author bsetfeng - * @since 1.0 - **/ -@Component -@Order(1) -@Slf4j -public class RuleEngineLogIndexInitialize { - - public RuleEngineLogIndexInitialize(ElasticSearchIndexManager indexManager) { - indexManager.putIndex(new DefaultElasticSearchIndexMetadata(RuleEngineLoggerIndexProvider.RULE_LOG.getIndex()) - .addProperty("createTime", new DateTimeType()) - .addProperty("level", new StringType()) - .addProperty("message", new StringType()) - .addProperty("nodeId", new StringType()) - .addProperty("instanceId", new StringType())) - .then( - indexManager.putIndex(new DefaultElasticSearchIndexMetadata(RuleEngineLoggerIndexProvider.RULE_EVENT_LOG.getIndex()) - .addProperty("createTime", new DateTimeType()) - .addProperty("event", new StringType()) - .addProperty("nodeId", new StringType()) - .addProperty("ruleData",new StringType()) - .addProperty("instanceId", new StringType())) - ) - .subscribe(); - } - -} diff --git a/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/configuration/RuleEngineProperties.java b/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/configuration/RuleEngineProperties.java new file mode 100755 index 00000000..6177b1d3 --- /dev/null +++ b/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/configuration/RuleEngineProperties.java @@ -0,0 +1,24 @@ +package org.jetlinks.community.rule.engine.configuration; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; + +@Getter +@Setter +@ConfigurationProperties(prefix = "rule.engine") +public class RuleEngineProperties { + + private String clusterName = "jetlinks"; + + private String serverId = "default"; + + private String serverName = "default"; + + /** + * 规则命令空间,相同命令空间的集群节点才会被调度 + */ + private String namespace; + + +} diff --git a/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/configuration/SpringClusterWorker.java b/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/configuration/SpringClusterWorker.java new file mode 100644 index 00000000..d1dae5a3 --- /dev/null +++ b/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/configuration/SpringClusterWorker.java @@ -0,0 +1,28 @@ +package org.jetlinks.community.rule.engine.configuration; + +import org.jetlinks.core.event.EventBus; +import org.jetlinks.rule.engine.api.task.TaskExecutorProvider; +import org.jetlinks.rule.engine.cluster.worker.ClusterWorker; +import org.jetlinks.rule.engine.cluster.worker.RuleIOManager; +import org.springframework.beans.factory.SmartInitializingSingleton; +import org.springframework.context.ApplicationContext; + +public class SpringClusterWorker extends ClusterWorker implements SmartInitializingSingleton { + private final ApplicationContext context; + + public SpringClusterWorker(String id, + String name, + EventBus eventBus, + RuleIOManager ioManager, + ApplicationContext context) { + super(id, name, eventBus, ioManager); + this.context = context; + } + + @Override + public void afterSingletonsInstantiated() { + context + .getBeansOfType(TaskExecutorProvider.class) + .forEach((beanName, provider) -> addExecutor(provider)); + } +} diff --git a/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/entity/TaskSnapshotEntity.java b/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/entity/TaskSnapshotEntity.java new file mode 100755 index 00000000..727bd6e2 --- /dev/null +++ b/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/entity/TaskSnapshotEntity.java @@ -0,0 +1,88 @@ +package org.jetlinks.community.rule.engine.entity; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Generated; +import lombok.Getter; +import lombok.Setter; +import org.hswebframework.ezorm.rdb.mapping.annotation.*; +import org.hswebframework.web.api.crud.entity.GenericEntity; +import org.hswebframework.web.bean.FastBeanCopier; +import org.hswebframework.web.crud.generator.Generators; +import org.jetlinks.rule.engine.api.scheduler.ScheduleJob; +import org.jetlinks.rule.engine.api.task.Task; +import org.jetlinks.rule.engine.api.task.TaskSnapshot; + +import javax.persistence.Column; +import javax.persistence.Index; +import javax.persistence.Table; +import java.sql.JDBCType; + +@Getter +@Setter +@Table(name = "rule_task_snapshot",indexes = { + @Index(name = "idx_rtsk_sid",columnList = "schedulerId"), + @Index(name = "idx_rtsk_inid",columnList = "instanceId,nodeId") +}) +@Comment("规则快照表") +@Generated +public class TaskSnapshotEntity extends GenericEntity { + + @Column(length = 64, nullable = false) + @Schema(description = "规则实例ID") + private String instanceId; + + @Column(length = 128, nullable = false) + @Schema(description = "调度ID") + private String schedulerId; + + @Column(length = 128, nullable = false) + @Schema(description = "执行ID") + private String workerId; + + @Column(length = 64, nullable = false) + @Schema(description = "节点ID") + private String nodeId; + + @Column(length = 64, nullable = false) + @Schema(description = "执行器") + private String executor; + + @Column + @Schema(description = "名称") + private String name; + + @Column + @JsonCodec + @ColumnType(jdbcType = JDBCType.CLOB, javaType = String.class) + @Schema(description = "调度任务") + private ScheduleJob job; + + @Column(nullable = false) + @DefaultValue(generator = Generators.CURRENT_TIME) + @Schema(description = "开始时间") + private Long startTime; + + @Column(nullable = false) + @DefaultValue(generator = Generators.CURRENT_TIME) + @Schema(description = "最近状态更新时间") + private Long lastStateTime; + + @Column(nullable = false) + @EnumCodec + @ColumnType(javaType = String.class) + @DefaultValue("shutdown") + @Schema(description = "状态") + private Task.State state; + + public TaskSnapshot toSnapshot(){ + return FastBeanCopier.copy(this,new TaskSnapshot()); + } + + public static TaskSnapshotEntity of(TaskSnapshot snapshot) { + TaskSnapshotEntity entity = FastBeanCopier.copy(snapshot, new TaskSnapshotEntity()); + entity.setExecutor(snapshot.getJob().getExecutor()); + entity.setNodeId(snapshot.getJob().getNodeId()); + + return entity; + } +} diff --git a/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/event/handler/RuleEngineLoggerIndexProvider.java b/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/event/handler/RuleEngineLoggerIndexProvider.java deleted file mode 100644 index 0844c61c..00000000 --- a/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/event/handler/RuleEngineLoggerIndexProvider.java +++ /dev/null @@ -1,21 +0,0 @@ -package org.jetlinks.community.rule.engine.event.handler; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import org.jetlinks.community.elastic.search.index.ElasticIndex; - -/** - * @author bsetfeng - * @since 1.0 - **/ -@Getter -@AllArgsConstructor -public enum RuleEngineLoggerIndexProvider implements ElasticIndex { - - RULE_LOG("rule-engine-execute-log", "_doc"), - RULE_EVENT_LOG("rule-engine-execute-event", "_doc"); - - private String index; - - private String type; -} diff --git a/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/event/handler/RuleLogHandler.java b/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/event/handler/RuleLogHandler.java deleted file mode 100644 index 49bcb3f8..00000000 --- a/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/event/handler/RuleLogHandler.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.jetlinks.community.rule.engine.event.handler; - -import com.alibaba.fastjson.JSONObject; -import lombok.extern.slf4j.Slf4j; -import org.hswebframework.web.bean.FastBeanCopier; -import org.jetlinks.community.elastic.search.service.ElasticSearchService; -import org.jetlinks.community.gateway.annotation.Subscribe; -import org.jetlinks.community.rule.engine.entity.RuleEngineExecuteEventInfo; -import org.jetlinks.core.event.TopicPayload; -import org.jetlinks.rule.engine.defaults.LogEvent; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.annotation.Order; -import org.springframework.stereotype.Component; -import reactor.core.publisher.Mono; - -@Component -@Slf4j -@Order(3) -public class RuleLogHandler { - - @Autowired - private ElasticSearchService elasticSearchService; - - @Subscribe("/rule-engine/*/*/event/${rule.engine.event.level:error}") - public Mono handleEvent(TopicPayload event) { - return elasticSearchService.commit(RuleEngineLoggerIndexProvider.RULE_EVENT_LOG, RuleEngineExecuteEventInfo.of(event)); - } - - @Subscribe("/rule-engine/*/*/logger/${rule.engine.logging.level:info,warn,error}") - public Mono handleLog(LogEvent event) { - JSONObject jsonObject = FastBeanCopier.copy(event, new JSONObject()); - jsonObject.put("createTime", System.currentTimeMillis()); - return elasticSearchService.commit(RuleEngineLoggerIndexProvider.RULE_LOG, jsonObject); - } -} diff --git a/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/executor/device/DeviceSelectorSpec.java b/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/executor/device/DeviceSelectorSpec.java index 276740ac..a79d9b21 100644 --- a/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/executor/device/DeviceSelectorSpec.java +++ b/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/executor/device/DeviceSelectorSpec.java @@ -17,8 +17,8 @@ import org.jetlinks.core.things.ThingsRegistry; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.NotNull; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; import java.util.List; import java.util.Map; diff --git a/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/io/EventBusRuleIOManager.java b/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/io/EventBusRuleIOManager.java new file mode 100644 index 00000000..ab2d725a --- /dev/null +++ b/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/io/EventBusRuleIOManager.java @@ -0,0 +1,117 @@ +package org.jetlinks.community.rule.engine.io; + +import org.apache.commons.collections4.CollectionUtils; +import org.jetlinks.core.cluster.ClusterManager; +import org.jetlinks.core.event.EventBus; +import org.jetlinks.core.event.Subscription; +import org.jetlinks.core.lang.SharedPathString; +import org.jetlinks.core.utils.RecyclerUtils; +import org.jetlinks.rule.engine.api.RuleData; +import org.jetlinks.rule.engine.api.scheduler.ScheduleJob; +import org.jetlinks.rule.engine.api.scope.GlobalScope; +import org.jetlinks.rule.engine.api.task.CompositeOutput; +import org.jetlinks.rule.engine.api.task.ConditionEvaluator; +import org.jetlinks.rule.engine.api.task.Input; +import org.jetlinks.rule.engine.api.task.Output; +import org.jetlinks.rule.engine.cluster.scope.ClusterGlobalScope; +import org.jetlinks.rule.engine.cluster.worker.RuleIOManager; +import org.jetlinks.rule.engine.defaults.EventBusEventOutput; +import org.jetlinks.rule.engine.defaults.EventBusOutput; +import reactor.core.Disposable; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.util.Collections; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +public class EventBusRuleIOManager implements RuleIOManager { + private final EventBus eventBus; + + private final ConditionEvaluator conditionEvaluator; + + private final GlobalScope globalScope; + + static { + RecyclerUtils.share("_r_e_in"); + RecyclerUtils.share("rule-engine"); + } + + static final SharedPathString baseEventAddress = SharedPathString.of("/_r_e_in/*/*"); + + public EventBusRuleIOManager(EventBus eventBus, ConditionEvaluator conditionEvaluator, ClusterManager clusterManager) { + this.eventBus = eventBus; + this.conditionEvaluator = conditionEvaluator; + this.globalScope = new ClusterGlobalScope(clusterManager); + } + + @Override + public Input createInput(ScheduleJob job) { + String instanceId = job.getInstanceId(); + String nodeId = job.getNodeId(); + return new Input() { + @Override + public Flux accept() { + return eventBus.subscribe( + Subscription + .builder() + .subscriberId("rule-engine:" + instanceId + ":" + nodeId) + .topics(createAddress(instanceId, nodeId)) + .features(Subscription.Feature.clusterSharedLocalFirstFeatures) + .build(), + RuleData.class); + } + + @Override + public Disposable accept(Function> listener) { + return eventBus.subscribe( + Subscription + .builder() + .subscriberId("rule-engine:" + instanceId + ":" + nodeId) + .topics(createAddress(instanceId, nodeId)) + .features(Subscription.Feature.clusterSharedLocalFirstFeatures) + .build(), + data -> listener.apply(data.decode(RuleData.class)).then()); + } + }; + } + + @Override + public Output createOutput(ScheduleJob job) { + return new EventBusOutput(job.getInstanceId(), eventBus, job.getOutputs(), conditionEvaluator) { + @Override + protected CharSequence createOutputAddress(String nodeId) { + return createAddress(instanceId, nodeId); + } + }; + } + + private CharSequence createAddress(String instanceId, String nodeId) { + // /_r_e_in/{instanceId}/{nodeId} + return baseEventAddress.replace(2, instanceId, 3, nodeId); + } + + @Override + public GlobalScope createScope() { + return globalScope; + } + + @Override + public Map createEvent(ScheduleJob job) { + String instanceId = job.getInstanceId(); + if (CollectionUtils.isEmpty(job.getEventOutputs())) { + return Collections.emptyMap(); + } + return job + .getEventOutputs() + .stream() + .map(event -> new EventBusEventOutput(instanceId, eventBus, event.getType(), event.getSource()) { + @Override + protected CharSequence createTopic(String node) { + return createAddress(instanceId, node); + } + }) + .collect(Collectors.groupingBy(EventBusEventOutput::getEvent, Collectors.collectingAndThen(Collectors.toList(), CompositeOutput::of))); + } +} diff --git a/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/log/RuleEngineLogService.java b/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/log/RuleEngineLogService.java new file mode 100755 index 00000000..7eab5a8e --- /dev/null +++ b/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/log/RuleEngineLogService.java @@ -0,0 +1,32 @@ +package org.jetlinks.community.rule.engine.log; + +import org.hswebframework.ezorm.core.param.QueryParam; +import org.hswebframework.web.api.crud.entity.PagerResult; +import org.jetlinks.community.rule.engine.entity.RuleEngineExecuteEventInfo; +import org.jetlinks.community.rule.engine.entity.RuleEngineExecuteLogInfo; +import reactor.core.publisher.Mono; + +/** + * 规则引擎日志服务,用于查询规则执行日志信息 + * + * @since 1.8 + */ +public interface RuleEngineLogService { + + /** + * 分页查询规则执行事件日志 + * + * @param queryParam 查询参数 + * @return 分页查询结果 + */ + Mono> queryEvent(QueryParam queryParam); + + /** + * 分页查询规则日志 + * + * @param queryParam 查询参数 + * @return 分页查询结果 + */ + Mono> queryLog(QueryParam queryParam); + +} diff --git a/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/log/TimeSeriesRuleEngineLogService.java b/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/log/TimeSeriesRuleEngineLogService.java new file mode 100755 index 00000000..9a23e40a --- /dev/null +++ b/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/log/TimeSeriesRuleEngineLogService.java @@ -0,0 +1,113 @@ +package org.jetlinks.community.rule.engine.log; + +import org.hswebframework.ezorm.core.param.QueryParam; +import org.hswebframework.web.api.crud.entity.PagerResult; +import org.hswebframework.web.bean.FastBeanCopier; +import org.jetlinks.core.event.TopicPayload; +import org.jetlinks.core.metadata.types.DateTimeType; +import org.jetlinks.core.metadata.types.StringType; +import org.jetlinks.community.ConfigMetadataConstants; +import org.jetlinks.community.gateway.annotation.Subscribe; +import org.jetlinks.community.rule.engine.entity.RuleEngineExecuteEventInfo; +import org.jetlinks.community.rule.engine.entity.RuleEngineExecuteLogInfo; +import org.jetlinks.community.timeseries.TimeSeriesData; +import org.jetlinks.community.timeseries.TimeSeriesManager; +import org.jetlinks.community.timeseries.TimeSeriesMetadata; +import org.jetlinks.community.timeseries.TimeSeriesMetric; +import org.jetlinks.community.utils.ObjectMappers; +import org.jetlinks.rule.engine.api.RuleData; +import org.jetlinks.rule.engine.defaults.LogEvent; +import org.springframework.beans.factory.SmartInitializingSingleton; +import reactor.core.publisher.Mono; + +import java.util.HashMap; +import java.util.Map; + +import static org.jetlinks.core.metadata.SimplePropertyMetadata.of; + +public class TimeSeriesRuleEngineLogService implements RuleEngineLogService, SmartInitializingSingleton { + + public static final TimeSeriesMetric RULE_LOG = TimeSeriesMetric.of("rule-engine-execute-log"); + public static final TimeSeriesMetric RULE_EVENT_LOG = TimeSeriesMetric.of("rule-engine-execute-event"); + + private final TimeSeriesManager timeSeriesManager; + + public TimeSeriesRuleEngineLogService(TimeSeriesManager timeSeriesManager) { + this.timeSeriesManager = timeSeriesManager; + } + + + /** + * @see org.jetlinks.rule.engine.api.RuleConstants.Event + */ + @Subscribe("/rule-engine/*/*/event/${rule.engine.event.level:error}") + public Mono handleEvent(TopicPayload event) { + + long now = System.currentTimeMillis(); + + Map data = new HashMap<>( + event.getTopicVars("/rule-engine/{instanceId}/{nodeId}/event/{event}") + ); + + RuleData ruleData = event.decode(RuleData.class); + data.put("id", ruleData.getId()); + data.put("contextId", ruleData.getContextId()); + data.put("ruleData", ObjectMappers.toJsonString(ruleData)); + data.put("createTime", now); + return timeSeriesManager + .getService(RULE_EVENT_LOG) + .commit(TimeSeriesData.of(now, data)); + } + + @Subscribe("/rule-engine/*/*/logger/${rule.engine.logging.level:info,warn,error}") + public Mono handleLog(LogEvent event) { + Map data = FastBeanCopier.copy(event, new HashMap<>()); + long now = System.currentTimeMillis(); + data.put("createTime", now); + return timeSeriesManager + .getService(RULE_LOG) + .commit(TimeSeriesData.of(event.getTimestamp(), data)); + } + + public Mono> queryEvent(QueryParam queryParam) { + return timeSeriesManager + .getService(RULE_EVENT_LOG) + .queryPager(queryParam, ts -> ts.as(RuleEngineExecuteEventInfo.class)); + } + + public Mono> queryLog(QueryParam queryParam) { + return timeSeriesManager + .getService(RULE_LOG) + .queryPager(queryParam, ts -> ts.as(RuleEngineExecuteLogInfo.class)); + } + + @Override + public void afterSingletonsInstantiated() { + timeSeriesManager + .registerMetadata( + TimeSeriesMetadata + .of(RULE_LOG, + of("createTime", "创建时间", DateTimeType.GLOBAL), + of("timestamp", "日志时间", DateTimeType.GLOBAL), + of("level", "日志级别", StringType.GLOBAL), + of("message", "消息", + new StringType().expand(ConfigMetadataConstants.maxLength, 8096L)), + of("nodeId", "规则节点ID", StringType.GLOBAL), + of("instanceId", "规则实例ID", StringType.GLOBAL) + )) + .then(timeSeriesManager + .registerMetadata( + TimeSeriesMetadata + .of(RULE_EVENT_LOG, + of("createTime", "创建时间", DateTimeType.GLOBAL), + of("timestamp", "日志时间", DateTimeType.GLOBAL), + of("event", "事件", StringType.GLOBAL), + of("ruleData", "数据", + new StringType().expand(ConfigMetadataConstants.maxLength, 8096L)), + of("nodeId", "规则节点ID", StringType.GLOBAL), + of("instanceId", "规则实例ID", StringType.GLOBAL) + )) + ) + .subscribe(); + } +} diff --git a/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/repository/LocalTaskSnapshotRepository.java b/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/repository/LocalTaskSnapshotRepository.java new file mode 100755 index 00000000..a234ca31 --- /dev/null +++ b/jetlinks-components/rule-engine-component/src/main/java/org/jetlinks/community/rule/engine/repository/LocalTaskSnapshotRepository.java @@ -0,0 +1,122 @@ +package org.jetlinks.community.rule.engine.repository; + +import org.hswebframework.ezorm.rdb.mapping.ReactiveRepository; +import org.jetlinks.community.rule.engine.entity.TaskSnapshotEntity; +import org.jetlinks.rule.engine.api.task.TaskSnapshot; +import org.jetlinks.rule.engine.cluster.TaskSnapshotRepository; +import org.reactivestreams.Publisher; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.util.Collection; + +public class LocalTaskSnapshotRepository implements TaskSnapshotRepository { + + private final ReactiveRepository repository; + + public LocalTaskSnapshotRepository(ReactiveRepository repository) { + this.repository = repository; + } + + @Override + public Flux findAllTask() { + return repository + .createQuery() + .fetch() + .map(TaskSnapshotEntity::toSnapshot); + } + + + @Override + public Flux findByInstanceId(String instanceId) { + return repository + .createQuery() + .where(TaskSnapshotEntity::getInstanceId, instanceId) + .fetch() + .map(TaskSnapshotEntity::toSnapshot); + } + + @Override + public Flux findByWorkerId(String workerId) { + return repository + .createQuery() + .where(TaskSnapshotEntity::getWorkerId, workerId) + .fetch() + .map(TaskSnapshotEntity::toSnapshot); + } + + @Override + public Flux findBySchedulerId(String schedulerId) { + return repository + .createQuery() + .where(TaskSnapshotEntity::getSchedulerId, schedulerId) + .fetch() + .map(TaskSnapshotEntity::toSnapshot); + } + + @Override + public Flux findBySchedulerIdNotIn(Collection schedulerId) { + return repository + .createQuery() + .where() + .notIn(TaskSnapshotEntity::getSchedulerId, schedulerId) + .fetch() + .map(TaskSnapshotEntity::toSnapshot); + } + + @Override + public Flux findByInstanceIdAndWorkerId(String instanceId, String workerId) { + return repository + .createQuery() + .where(TaskSnapshotEntity::getInstanceId, instanceId) + .and(TaskSnapshotEntity::getWorkerId, workerId) + .fetch() + .map(TaskSnapshotEntity::toSnapshot); + } + + @Override + public Flux findByInstanceIdAndNodeId(String instanceId, String nodeId) { + return repository + .createQuery() + .where(TaskSnapshotEntity::getInstanceId, instanceId) + .and(TaskSnapshotEntity::getNodeId, nodeId) + .fetch() + .map(TaskSnapshotEntity::toSnapshot); + } + + @Override + public Mono saveTaskSnapshots(Publisher snapshots) { + return Flux + .from(snapshots) + .map(TaskSnapshotEntity::of) + .buffer(200) + .concatMap(repository::save) + .then(); + } + + @Override + public Mono removeTaskByInstanceId(String instanceId) { + return repository + .createDelete() + .where(TaskSnapshotEntity::getInstanceId, instanceId) + .execute() + .then(); + } + + @Override + public Mono removeTaskByInstanceIdAndNodeId(String instanceId, String nodeId) { + return repository + .createDelete() + .where(TaskSnapshotEntity::getInstanceId, instanceId) + .and(TaskSnapshotEntity::getNodeId, nodeId) + .execute() + .then(); + } + + @Override + public Mono removeTaskById(String id) { + return repository + .deleteById(id) + .then(); + } +} diff --git a/jetlinks-components/rule-engine-component/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/jetlinks-components/rule-engine-component/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports new file mode 100644 index 00000000..d816bba6 --- /dev/null +++ b/jetlinks-components/rule-engine-component/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports @@ -0,0 +1 @@ +org.jetlinks.community.rule.engine.configuration.RuleEngineConfiguration \ No newline at end of file diff --git a/jetlinks-components/script-component/pom.xml b/jetlinks-components/script-component/pom.xml index c4ba0302..e034b2ef 100644 --- a/jetlinks-components/script-component/pom.xml +++ b/jetlinks-components/script-component/pom.xml @@ -5,7 +5,7 @@ jetlinks-components org.jetlinks.community - 2.3.0-SNAPSHOT + 2.10.0-SNAPSHOT ../pom.xml 4.0.0 @@ -19,12 +19,11 @@ guava - - - - - - + + org.openjdk.nashorn + nashorn-core + 15.6 + org.jetlinks diff --git a/jetlinks-components/script-component/src/test/java/org/jetlinks/community/script/JavaScriptFactoryTest.java b/jetlinks-components/script-component/src/test/java/org/jetlinks/community/script/JavaScriptFactoryTest.java index 3de0bb78..484482cb 100644 --- a/jetlinks-components/script-component/src/test/java/org/jetlinks/community/script/JavaScriptFactoryTest.java +++ b/jetlinks-components/script-component/src/test/java/org/jetlinks/community/script/JavaScriptFactoryTest.java @@ -1,8 +1,6 @@ package org.jetlinks.community.script; -import jdk.nashorn.internal.objects.Global; import lombok.SneakyThrows; -import lombok.extern.java.Log; import org.jetlinks.community.script.jsr223.JavaScriptFactory; import org.junit.jupiter.api.Test; diff --git a/jetlinks-components/tdengine-component/pom.xml b/jetlinks-components/tdengine-component/pom.xml index c537f14f..f4c13cfb 100755 --- a/jetlinks-components/tdengine-component/pom.xml +++ b/jetlinks-components/tdengine-component/pom.xml @@ -5,7 +5,7 @@ jetlinks-components org.jetlinks.community - 2.3.0-SNAPSHOT + 2.10.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/jetlinks-components/tdengine-component/src/main/java/org/jetlinks/community/tdengine/TDEngineUtils.java b/jetlinks-components/tdengine-component/src/main/java/org/jetlinks/community/tdengine/TDEngineUtils.java index 8e1d5275..e17915fd 100644 --- a/jetlinks-components/tdengine-component/src/main/java/org/jetlinks/community/tdengine/TDEngineUtils.java +++ b/jetlinks-components/tdengine-component/src/main/java/org/jetlinks/community/tdengine/TDEngineUtils.java @@ -27,7 +27,7 @@ public class TDEngineUtils { .doOnNext(str -> { throw new TDengineException(null, str); }) - .switchIfEmpty(Mono.error(() -> new TDengineException(null, response.statusCode().getReasonPhrase()))) + .switchIfEmpty(Mono.error(() -> new TDengineException(null, response.statusCode().toString()))) .then(Mono.empty()); } diff --git a/jetlinks-components/tdengine-component/src/main/java/org/jetlinks/community/tdengine/TDengineProperties.java b/jetlinks-components/tdengine-component/src/main/java/org/jetlinks/community/tdengine/TDengineProperties.java index 8e2950e2..a65794ad 100755 --- a/jetlinks-components/tdengine-component/src/main/java/org/jetlinks/community/tdengine/TDengineProperties.java +++ b/jetlinks-components/tdengine-component/src/main/java/org/jetlinks/community/tdengine/TDengineProperties.java @@ -24,7 +24,7 @@ import reactor.netty.resources.LoopResources; import reactor.netty.tcp.TcpClient; import javax.sql.DataSource; -import javax.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotBlank; import java.net.URI; import java.time.Duration; import java.util.ArrayList; diff --git a/jetlinks-components/things-component/pom.xml b/jetlinks-components/things-component/pom.xml index 55e7c354..c17c6472 100644 --- a/jetlinks-components/things-component/pom.xml +++ b/jetlinks-components/things-component/pom.xml @@ -5,19 +5,13 @@ jetlinks-components org.jetlinks.community - 2.3.0-SNAPSHOT + 2.10.0-SNAPSHOT ../pom.xml 4.0.0 things-component - - 8 - 8 - UTF-8 - - org.jetlinks diff --git a/jetlinks-components/things-component/src/main/java/org/jetlinks/community/things/configuration/AutoRegisterThingsRegistry.java b/jetlinks-components/things-component/src/main/java/org/jetlinks/community/things/configuration/AutoRegisterThingsRegistry.java index c1b05f29..1d3a32e2 100644 --- a/jetlinks-components/things-component/src/main/java/org/jetlinks/community/things/configuration/AutoRegisterThingsRegistry.java +++ b/jetlinks-components/things-component/src/main/java/org/jetlinks/community/things/configuration/AutoRegisterThingsRegistry.java @@ -3,17 +3,28 @@ package org.jetlinks.community.things.configuration; import org.jetlinks.core.things.DefaultThingsRegistry; import org.jetlinks.core.things.ThingsRegistrySupport; import org.springframework.beans.BeansException; +import org.springframework.beans.factory.SmartInitializingSingleton; import org.springframework.beans.factory.config.BeanPostProcessor; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; import javax.annotation.Nonnull; -public class AutoRegisterThingsRegistry extends DefaultThingsRegistry implements BeanPostProcessor { +public class AutoRegisterThingsRegistry extends DefaultThingsRegistry + implements SmartInitializingSingleton, ApplicationContextAware { + + private ApplicationContext context; @Override - public Object postProcessAfterInitialization(@Nonnull Object bean,@Nonnull String beanName) throws BeansException { - if(bean instanceof ThingsRegistrySupport){ - addSupport(((ThingsRegistrySupport) bean)); + public void afterSingletonsInstantiated() { + if (context != null) { + context.getBeanProvider(ThingsRegistrySupport.class) + .forEach(this::addSupport); } - return BeanPostProcessor.super.postProcessAfterInitialization(bean, beanName); + } + + @Override + public void setApplicationContext(@Nonnull ApplicationContext applicationContext) throws BeansException { + this.context = applicationContext; } } diff --git a/jetlinks-components/things-component/src/main/java/org/jetlinks/community/things/data/PropertyAggregation.java b/jetlinks-components/things-component/src/main/java/org/jetlinks/community/things/data/PropertyAggregation.java index d63a7c10..716c8e14 100644 --- a/jetlinks-components/things-component/src/main/java/org/jetlinks/community/things/data/PropertyAggregation.java +++ b/jetlinks-components/things-component/src/main/java/org/jetlinks/community/things/data/PropertyAggregation.java @@ -9,8 +9,8 @@ import org.apache.commons.lang3.StringUtils; import org.hswebframework.web.validator.ValidatorUtils; import org.jetlinks.community.timeseries.query.Aggregation; -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.NotNull; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; @Getter @Setter diff --git a/jetlinks-components/things-component/src/main/java/org/jetlinks/community/things/data/TableSafeMetricBuilder.java b/jetlinks-components/things-component/src/main/java/org/jetlinks/community/things/data/TableSafeMetricBuilder.java new file mode 100644 index 00000000..07d80acd --- /dev/null +++ b/jetlinks-components/things-component/src/main/java/org/jetlinks/community/things/data/TableSafeMetricBuilder.java @@ -0,0 +1,68 @@ +package org.jetlinks.community.things.data; + +import lombok.AllArgsConstructor; +import org.jetlinks.community.things.data.operations.MetricBuilder; +import org.jetlinks.core.config.ConfigKey; +import org.jetlinks.community.things.utils.ThingsDatabaseUtils; + +import javax.annotation.Nonnull; +import java.util.Optional; + +@AllArgsConstructor(staticName = "of") +public class TableSafeMetricBuilder implements MetricBuilder{ + private final MetricBuilder target; + + @Override + public Optional option(ConfigKey key) { + return target.option(key); + } + + @Override + public String getThingIdProperty() { + return target.getThingIdProperty(); + } + + @Override + public String getTemplateIdProperty() { + return target.getTemplateIdProperty(); + } + + @Override + public String createEventAllInOneMetric(@Nonnull String thingType, @Nonnull String thingTemplateId, String thingId) { + return ThingsDatabaseUtils.createTableName(target.createEventAllInOneMetric(thingType, thingTemplateId, thingId)); + } + + @Override + public String createPropertyMetric(@Nonnull String thingType, + @Nonnull String thingTemplateId, + String thingId, String group) { + return ThingsDatabaseUtils.createTableName(target.createPropertyMetric(thingType, thingTemplateId, thingId, group)); + } + + @Override + public String createEventMetric(@Nonnull String thingType, @Nonnull String thingTemplateId, String thingId, @Nonnull String eventId) { + return ThingsDatabaseUtils.createTableName( + target.createEventMetric(thingType, thingTemplateId, thingId, eventId) + ); + } + + @Override + public String createLogMetric(@Nonnull String thingType, @Nonnull String thingTemplateId, String thingId) { + return ThingsDatabaseUtils.createTableName( + target.createLogMetric(thingType, thingTemplateId, thingId) + ); + } + + @Override + public String createLogMetric(@Nonnull String thingType) { + return ThingsDatabaseUtils.createTableName(target.createLogMetric(thingType)); + } + + @Override + public String createPropertyMetric(@Nonnull String thingType, @Nonnull String thingTemplateId, String thingId) { + return ThingsDatabaseUtils.createTableName( + target.createPropertyMetric(thingType, thingTemplateId, thingId) + ); + } + +} diff --git a/jetlinks-components/things-component/src/main/java/org/jetlinks/community/things/data/ThingsDataUtils.java b/jetlinks-components/things-component/src/main/java/org/jetlinks/community/things/data/ThingsDataUtils.java new file mode 100644 index 00000000..9e0783ae --- /dev/null +++ b/jetlinks-components/things-component/src/main/java/org/jetlinks/community/things/data/ThingsDataUtils.java @@ -0,0 +1,67 @@ +package org.jetlinks.community.things.data; + +import org.jetlinks.community.timeseries.utils.TimeSeriesUtils; +import org.jetlinks.community.Interval; +import org.jetlinks.community.timeseries.utils.TimeSeriesUtils; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; +import org.joda.time.Instant; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; + +import java.util.*; +import java.util.function.BiFunction; + +public class ThingsDataUtils { + + public static Map findAggregationData(long ts, + NavigableMap> container) { + + Map prepare = container.get(ts); + if (prepare != null) { + return prepare; + } + Map.Entry> entry = container.floorEntry(ts); + if (entry != null) { + return entry.getValue(); + } + return null; + } + + public static NavigableMap> prepareAggregationData(AggregationRequest request, + PropertyAggregation... properties) { + + return prepareAggregationData(request, TimeSeriesUtils::truncateTime, properties); + } + + public static NavigableMap> prepareAggregationData(AggregationRequest request, + BiFunction timeTruncate, + PropertyAggregation... properties) { + NavigableMap> data = new TreeMap<>(Comparator.comparingLong(l -> -l)); + Map valueMap = new HashMap<>(); + for (PropertyAggregation property : properties) { + valueMap.put(property.getAlias(), property.getDefaultValue()); + } + if (request.getInterval() == null) { + data.put(0L,valueMap); + return data; + } + DateTimeFormatter formatter = DateTimeFormat.forPattern(request.getFormat()); + + long startWith = request.getFrom().getTime(); + long endWith = request.getTo().getTime(); + + Iterable iterable = request.getInterval().iterate(startWith, endWith); + for (Long time : iterable) { + time = timeTruncate.apply(time, request.getInterval()); + Map tempData = new HashMap<>(valueMap); + //使用joad-time，有的格式java.time不支持... + String formatValue = new DateTime(new Instant(time), DateTimeZone.getDefault()).toString(formatter); + tempData.put("time", formatValue); + data.put(time, tempData); + } + + return data; + } + +} diff --git a/jetlinks-components/things-component/src/main/java/org/jetlinks/community/things/data/operations/DataSettings.java b/jetlinks-components/things-component/src/main/java/org/jetlinks/community/things/data/operations/DataSettings.java index 0a48d95c..05f8d9c9 100644 --- a/jetlinks-components/things-component/src/main/java/org/jetlinks/community/things/data/operations/DataSettings.java +++ b/jetlinks-components/things-component/src/main/java/org/jetlinks/community/things/data/operations/DataSettings.java @@ -28,6 +28,9 @@ public class DataSettings { public static class Property { //是否只保存属性上报消息 private boolean onlySaveReport = false; + //同时查询多个属性时使用聚合查询 + private boolean queryPropertiesAggregation = true; + } @Getter diff --git a/jetlinks-components/things-component/src/main/java/org/jetlinks/community/things/data/operations/TableSafeMetricBuilder.java b/jetlinks-components/things-component/src/main/java/org/jetlinks/community/things/data/operations/TableSafeMetricBuilder.java new file mode 100644 index 00000000..eb6146ef --- /dev/null +++ b/jetlinks-components/things-component/src/main/java/org/jetlinks/community/things/data/operations/TableSafeMetricBuilder.java @@ -0,0 +1,67 @@ +package org.jetlinks.community.things.data.operations; + +import lombok.AllArgsConstructor; +import org.jetlinks.core.config.ConfigKey; +import org.jetlinks.community.things.utils.ThingsDatabaseUtils; + +import javax.annotation.Nonnull; +import java.util.Optional; + +@AllArgsConstructor(staticName = "of") +public class TableSafeMetricBuilder implements MetricBuilder{ + private final MetricBuilder target; + + @Override + public Optional option(ConfigKey key) { + return target.option(key); + } + + @Override + public String getThingIdProperty() { + return target.getThingIdProperty(); + } + + @Override + public String getTemplateIdProperty() { + return target.getTemplateIdProperty(); + } + + @Override + public String createEventAllInOneMetric(@Nonnull String thingType, @Nonnull String thingTemplateId, String thingId) { + return ThingsDatabaseUtils.createTableName(target.createEventAllInOneMetric(thingType, thingTemplateId, thingId)); + } + + @Override + public String createPropertyMetric(@Nonnull String thingType, + @Nonnull String thingTemplateId, + String thingId, String group) { + return ThingsDatabaseUtils.createTableName(target.createPropertyMetric(thingType, thingTemplateId, thingId, group)); + } + + @Override + public String createEventMetric(@Nonnull String thingType, @Nonnull String thingTemplateId, String thingId, @Nonnull String eventId) { + return ThingsDatabaseUtils.createTableName( + target.createEventMetric(thingType, thingTemplateId, thingId, eventId) + ); + } + + @Override + public String createLogMetric(@Nonnull String thingType, @Nonnull String thingTemplateId, String thingId) { + return ThingsDatabaseUtils.createTableName( + target.createLogMetric(thingType, thingTemplateId, thingId) + ); + } + + @Override + public String createLogMetric(@Nonnull String thingType) { + return ThingsDatabaseUtils.createTableName(target.createLogMetric(thingType)); + } + + @Override + public String createPropertyMetric(@Nonnull String thingType, @Nonnull String thingTemplateId, String thingId) { + return ThingsDatabaseUtils.createTableName( + target.createPropertyMetric(thingType, thingTemplateId, thingId) + ); + } + +} diff --git a/jetlinks-components/things-component/src/main/java/org/jetlinks/community/things/utils/ThingsDatabaseUtils.java b/jetlinks-components/things-component/src/main/java/org/jetlinks/community/things/utils/ThingsDatabaseUtils.java index 4f5195c0..f0e1b5c2 100644 --- a/jetlinks-components/things-component/src/main/java/org/jetlinks/community/things/utils/ThingsDatabaseUtils.java +++ b/jetlinks-components/things-component/src/main/java/org/jetlinks/community/things/utils/ThingsDatabaseUtils.java @@ -3,17 +3,18 @@ package org.jetlinks.community.things.utils; import org.hswebframework.ezorm.core.ValueCodec; import org.hswebframework.ezorm.core.param.Term; import org.hswebframework.ezorm.core.param.TermType; -import org.hswebframework.ezorm.rdb.codec.BooleanValueCodec; -import org.hswebframework.ezorm.rdb.codec.ClobValueCodec; -import org.hswebframework.ezorm.rdb.codec.JsonValueCodec; -import org.hswebframework.ezorm.rdb.codec.NumberValueCodec; +import org.hswebframework.ezorm.rdb.codec.*; import org.hswebframework.ezorm.rdb.metadata.RDBColumnMetadata; -import org.jetlinks.community.ConfigMetadataConstants; -import org.jetlinks.community.utils.ConverterUtils; +import org.hswebframework.web.utils.DigestUtils; +import org.jetlinks.core.metadata.Converter; import org.jetlinks.core.metadata.DataType; import org.jetlinks.core.metadata.PropertyMetadata; +import org.jetlinks.core.metadata.SimplePropertyMetadata; import org.jetlinks.core.metadata.types.*; import org.jetlinks.core.utils.StringBuilderUtils; +import org.jetlinks.community.ConfigMetadataConstants; +import org.jetlinks.community.utils.ConverterUtils; +import org.jetlinks.community.utils.TimeUtils; import org.jetlinks.reactor.ql.utils.CastUtils; import org.jetlinks.supports.official.DeviceMetadataParser; import org.springframework.core.ResolvableType; @@ -168,6 +169,10 @@ public class ThingsDatabaseUtils { return convertColumn(metadata, new RDBColumnMetadata()); } + public static PropertyMetadata convertMetadata(RDBColumnMetadata column) { + return SimplePropertyMetadata.of(column.getAlias(), column.getComment(), convertDataType(column)); + } + public static DataType convertDataType(RDBColumnMetadata column) { DataType type; if (column.getJavaType() != null) { @@ -182,6 +187,24 @@ public class ThingsDatabaseUtils { private final static Base64.Encoder tableEncoder = Base64.getUrlEncoder().withoutPadding(); + + public static String createTableName(String prefix, String... suffixes) { + int len = prefix.length(); + + for (String suffix : suffixes) { + len += suffix.length(); + } + + if (len >= 64 - suffixes.length) { + //表名太长，使用短编码 + return createTableName0( + prefix, tableEncoder.encodeToString(DigestUtils.md5(String.join("_", suffixes))) + ); + } + + return createTableName0(prefix, suffixes); + } + private static String createTableName0(String prefix, String... suffixes) { return StringBuilderUtils .buildString(prefix, suffixes, (_prefix, _suffixes, builder) -> { @@ -208,6 +231,32 @@ public class ThingsDatabaseUtils { } } + public static Object tryConvertTermValue(DataType type, Object value) { + if (type instanceof DateTimeType) { + return TimeUtils.convertToDate(value).getTime(); + } else if (type instanceof Converter) { + return ((Converter) type).convert(value); + } + return value; + } + + public static void tryConvertTermValue(DataType type, + Term term, + BiFunction tryConvertTermValue) { + tryConvertTermValue(type, + term, + ThingsDatabaseUtils::isDoNotConvertValue, + ThingsDatabaseUtils::maybeList, + tryConvertTermValue); + } + + public static void tryConvertTermValue(DataType type, + Term term) { + tryConvertTermValue(type, + term, + ThingsDatabaseUtils::tryConvertTermValue); + } + public static void tryConvertTermValue(DataType type, Term term, BiPredicate isDoNotConvertValue, diff --git a/jetlinks-components/timescaledb-component/pom.xml b/jetlinks-components/timescaledb-component/pom.xml new file mode 100644 index 00000000..65d2eb6f --- /dev/null +++ b/jetlinks-components/timescaledb-component/pom.xml @@ -0,0 +1,46 @@ + + + 4.0.0 + + org.jetlinks.community + jetlinks-components + 2.10.0-SNAPSHOT + + + timescaledb-component + + + UTF-8 + + + + + org.jetlinks.community + things-component + + + + org.jetlinks.community + timeseries-component + + + + io.r2dbc + r2dbc-spi + + + + org.postgresql + r2dbc-postgresql + + + + org.jetlinks.community + datasource-component + ${project.version} + + + + \ No newline at end of file diff --git a/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/TimescaleDBDataWriter.java b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/TimescaleDBDataWriter.java new file mode 100644 index 00000000..ff2cf432 --- /dev/null +++ b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/TimescaleDBDataWriter.java @@ -0,0 +1,14 @@ +package org.jetlinks.community.timescaledb; + +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.util.Map; + +public interface TimescaleDBDataWriter { + + Mono save(String metric, Map data); + + Mono save(String metric, Flux> data); + +} diff --git a/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/TimescaleDBOperations.java b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/TimescaleDBOperations.java new file mode 100644 index 00000000..68f9c440 --- /dev/null +++ b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/TimescaleDBOperations.java @@ -0,0 +1,11 @@ +package org.jetlinks.community.timescaledb; + +import org.hswebframework.ezorm.rdb.operator.DatabaseOperator; + +public interface TimescaleDBOperations { + + DatabaseOperator database(); + + TimescaleDBDataWriter writer(); + +} diff --git a/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/TimescaleDBProperties.java b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/TimescaleDBProperties.java new file mode 100644 index 00000000..dd7bc533 --- /dev/null +++ b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/TimescaleDBProperties.java @@ -0,0 +1,42 @@ +package org.jetlinks.community.timescaledb; + +import lombok.Getter; +import lombok.Setter; +import org.jetlinks.community.buffer.BufferProperties; +import org.jetlinks.community.timescaledb.impl.DefaultTimescaleDBDataWriter; +import org.springframework.boot.autoconfigure.r2dbc.R2dbcProperties; +import org.springframework.boot.context.properties.ConfigurationProperties; + +@ConfigurationProperties(prefix = "timescaledb") +@Getter +@Setter +public class TimescaleDBProperties { + + private boolean enabled = false; + + //是否共享spring容器中的连接 + //需要平台也使用timescaledb + private boolean sharedSpring = false; + + //当sharedSpring未false时,使用此连接配置. + private R2dbcProperties r2dbc = new R2dbcProperties(); + + //数据库的schema + private String schema = "public"; + + /** + * 写入缓冲区配置 + * + * @see DefaultTimescaleDBDataWriter + */ + private BufferProperties writeBuffer = new BufferProperties(); + + public TimescaleDBProperties() { + writeBuffer.setFilePath("./data/timescaledb-buffer"); + writeBuffer.setSize(1000); + writeBuffer.setParallelism(4); + r2dbc.getPool().setMaxSize(64); + } + + +} diff --git a/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/TimescaleDBUtils.java b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/TimescaleDBUtils.java new file mode 100644 index 00000000..964e0873 --- /dev/null +++ b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/TimescaleDBUtils.java @@ -0,0 +1,94 @@ +package org.jetlinks.community.timescaledb; + +import org.hswebframework.ezorm.rdb.mapping.defaults.record.Record; +import org.hswebframework.ezorm.rdb.metadata.RDBColumnMetadata; +import org.hswebframework.ezorm.rdb.operator.dml.query.NativeSelectColumn; +import org.hswebframework.ezorm.rdb.operator.dml.query.SelectColumn; +import org.hswebframework.ezorm.rdb.supports.postgres.JsonbType; +import org.jetlinks.core.metadata.DataType; +import org.jetlinks.core.metadata.PropertyMetadata; +import org.jetlinks.core.metadata.types.ArrayType; +import org.jetlinks.core.metadata.types.ObjectType; +import org.jetlinks.community.Interval; +import org.jetlinks.community.things.data.ThingsDataConstants; +import org.jetlinks.community.things.utils.ThingsDatabaseUtils; +import org.jetlinks.community.timescaledb.metadata.JsonbValueCodec; +import org.jetlinks.community.timeseries.TimeSeriesData; +import org.jetlinks.community.timeseries.query.Aggregation; +import org.jetlinks.community.utils.ObjectMappers; +import org.jetlinks.reactor.ql.utils.CastUtils; + +public class TimescaleDBUtils { + + + public static String getTableName(String name) { + return ThingsDatabaseUtils.createTableName(name); + } + + public static NativeSelectColumn createTimeGroupColumn(long startWith, Interval interval) { + + String unit = interval.getNumber().intValue() + " " + interval + .getUnit() + .name() + .toLowerCase(); + + return NativeSelectColumn + .of("time_bucket('" + unit + "',timestamp)"); + } + + public static TimeSeriesData convertToTimeSeriesData(Record record) { + return TimeSeriesData.of( + record.get(ThingsDataConstants.COLUMN_TIMESTAMP) + .map(val -> CastUtils.castNumber(val).longValue()) + .orElseGet(System::currentTimeMillis), + record + ); + } + + public static void customColumn(PropertyMetadata metadata, RDBColumnMetadata column) { + DataType type = metadata.getValueType(); + + if (type instanceof ArrayType) { + column.setType(new JsonbType()); + column.setValueCodec(new JsonbValueCodec(true)); + } else if (type instanceof ObjectType) { + column.setType(new JsonbType()); + column.setValueCodec(new JsonbValueCodec(false)); + } + + } + + + public static void applyAggColumn(Aggregation aggregation, SelectColumn column) { + String function; + switch (aggregation) { + case COUNT: + function = "count"; + break; + case AVG: + function = "avg"; + break; + case FIRST: + case TOP: + case MAX: + function = "max"; + break; + case LAST: + case MIN: + function = "min"; + break; + case SUM: + function = "sum"; + break; + case DISTINCT_COUNT: + function = "count"; + column.option("distinct", true); + break; + default: + throw new UnsupportedOperationException("不支持的聚合函数:" + aggregation); + } + column.setFunction(function); + } + + +} diff --git a/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/configuration/TimescaleDBConfiguration.java b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/configuration/TimescaleDBConfiguration.java new file mode 100644 index 00000000..422e90b5 --- /dev/null +++ b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/configuration/TimescaleDBConfiguration.java @@ -0,0 +1,23 @@ +package org.jetlinks.community.timescaledb.configuration; + +import org.jetlinks.community.timescaledb.TimescaleDBProperties; +import org.jetlinks.community.timescaledb.impl.DefaultTimescaleDBOperations; +import org.springframework.boot.autoconfigure.AutoConfiguration; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; + +@AutoConfiguration +@EnableConfigurationProperties(TimescaleDBProperties.class) +@ConditionalOnProperty(prefix = "timescaledb", name = "enabled", havingValue = "true") +public class TimescaleDBConfiguration { + + + @Bean(destroyMethod = "shutdown",initMethod = "init") + public DefaultTimescaleDBOperations timescaleDBOperations(TimescaleDBProperties properties) { + + return new DefaultTimescaleDBOperations(properties); + } + + +} diff --git a/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/configuration/TimescaleDBThingsDataConfiguration.java b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/configuration/TimescaleDBThingsDataConfiguration.java new file mode 100644 index 00000000..a1813e01 --- /dev/null +++ b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/configuration/TimescaleDBThingsDataConfiguration.java @@ -0,0 +1,39 @@ +package org.jetlinks.community.timescaledb.configuration; + +import org.jetlinks.core.things.ThingsRegistry; +import org.jetlinks.community.things.data.ThingsDataRepositoryStrategy; +import org.jetlinks.community.timescaledb.TimescaleDBOperations; +import org.jetlinks.community.timescaledb.thing.TimescaleDBColumnModeStrategy; +import org.jetlinks.community.timescaledb.thing.TimescaleDBRowModeStrategy; +import org.jetlinks.community.timescaledb.thing.TimescaleDBThingsDataProperties; +import org.springframework.boot.autoconfigure.AutoConfiguration; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; + +@AutoConfiguration(after = TimescaleDBConfiguration.class) +@ConditionalOnBean({TimescaleDBOperations.class, ThingsRegistry.class}) +@ConditionalOnClass(ThingsDataRepositoryStrategy.class) +@ConditionalOnProperty(prefix = "timescaledb.things-data", name = "enabled", havingValue = "true", matchIfMissing = true) +@EnableConfigurationProperties(TimescaleDBThingsDataProperties.class) +public class TimescaleDBThingsDataConfiguration { + + + @Bean + public TimescaleDBRowModeStrategy timescaleDBRowModeStrategy(ThingsRegistry registry, + TimescaleDBOperations timescaleDBOperations, + TimescaleDBThingsDataProperties properties) { + return new TimescaleDBRowModeStrategy(registry, timescaleDBOperations, properties); + } + + @Bean + public TimescaleDBColumnModeStrategy timescaleDBColumnModeStrategy(ThingsRegistry registry, + TimescaleDBOperations timescaleDBOperations, + TimescaleDBThingsDataProperties properties) { + return new TimescaleDBColumnModeStrategy(registry, timescaleDBOperations, properties); + } + + +} diff --git a/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/configuration/TimescaleDBTimeSeriesConfiguration.java b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/configuration/TimescaleDBTimeSeriesConfiguration.java new file mode 100644 index 00000000..b95daa5b --- /dev/null +++ b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/configuration/TimescaleDBTimeSeriesConfiguration.java @@ -0,0 +1,28 @@ +package org.jetlinks.community.timescaledb.configuration; + +import org.jetlinks.community.timescaledb.TimescaleDBOperations; +import org.jetlinks.community.timescaledb.timeseries.TimescaleDBTimeSeriesManager; +import org.jetlinks.community.timescaledb.timeseries.TimescaleDBTimeSeriesProperties; +import org.springframework.boot.autoconfigure.AutoConfiguration; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Primary; + +@AutoConfiguration(after = TimescaleDBConfiguration.class) +@ConditionalOnBean(TimescaleDBOperations.class) +@ConditionalOnProperty(prefix = "timescaledb.time-series", name = "enabled", havingValue = "true", matchIfMissing = true) +@EnableConfigurationProperties(TimescaleDBTimeSeriesProperties.class) +public class TimescaleDBTimeSeriesConfiguration { + + + @Bean + @Primary + public TimescaleDBTimeSeriesManager timescaleDBTimeSeriesManager(TimescaleDBOperations operations, + TimescaleDBTimeSeriesProperties properties) { + return new TimescaleDBTimeSeriesManager(properties, operations); + } + + +} diff --git a/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/impl/DefaultTimescaleDBDataWriter.java b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/impl/DefaultTimescaleDBDataWriter.java new file mode 100644 index 00000000..c760de07 --- /dev/null +++ b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/impl/DefaultTimescaleDBDataWriter.java @@ -0,0 +1,183 @@ +package org.jetlinks.community.timescaledb.impl; + +import com.google.common.collect.Collections2; +import com.google.common.collect.Maps; +import io.r2dbc.spi.R2dbcException; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.hswebframework.ezorm.rdb.metadata.RDBTableMetadata; +import org.hswebframework.ezorm.rdb.operator.DatabaseOperator; +import org.hswebframework.web.exception.BusinessException; +import org.jetlinks.core.utils.Reactors; +import org.jetlinks.core.utils.SerializeUtils; +import org.jetlinks.community.buffer.BufferProperties; +import org.jetlinks.community.buffer.BufferSettings; +import org.jetlinks.community.buffer.Buffered; +import org.jetlinks.community.buffer.PersistenceBuffer; +import org.jetlinks.community.timescaledb.TimescaleDBDataWriter; +import org.jetlinks.community.utils.ErrorUtils; +import org.slf4j.Logger; +import org.springframework.boot.CommandLineRunner; +import org.springframework.boot.SpringApplication; +import org.springframework.dao.DataAccessResourceFailureException; +import org.springframework.dao.QueryTimeoutException; +import org.springframework.transaction.CannotCreateTransactionException; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.io.Externalizable; +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; + +@Slf4j +public class DefaultTimescaleDBDataWriter implements TimescaleDBDataWriter, CommandLineRunner { + + final BufferProperties properties; + final PersistenceBuffer buffer; + + final DatabaseOperator database; + + public DefaultTimescaleDBDataWriter(DatabaseOperator database, + BufferProperties properties) { + this.database = database; + this.properties = properties; + this.buffer = new PersistenceBuffer<>( + BufferSettings.create(properties), + Buffer::new, + this::save0) + .name("timescale-db-things-writer"); + } + + public void init() { + buffer.init(); + } + + public void start() { + buffer.start(); + } + + public void stop() { + buffer.stop(); + } + + public void shutdown() { + buffer.dispose(); + } + + + @Override + public Mono save(String metric, Map data) { + return buffer + .writeAsync(new Buffer(metric, data)); + } + + @Override + public Mono save(String metric, Flux> data) { + return data + .buffer(200) + .concatMap(list -> save0(metric, list)) + .then(); + } + + public Mono save0(Collection> buffers, PersistenceBuffer.FlushContext context) { + + Map>> grouped = buffers + .stream() + .collect(Collectors.groupingBy(buffed -> buffed.getData().metric, Collectors.toList())); + + return Flux + .fromIterable(grouped.entrySet()) + .flatMap(e -> this + .save0(e.getKey(), + Collections2.transform(e.getValue(), buffered -> buffered.getData().getData())) + .onErrorResume(err -> { + context.error(err); + if (needRetry(err)) { + for (Buffered buffered : e.getValue()) { + if (properties.isExceededRetryCount(buffered.getRetryTimes())) { + buffered.dead(); + } else { + buffered.retry(true); + } + } + } else { + log.warn("save timescaledb data error [{}]", e.getKey(), err); + for (Buffered buffered : e.getValue()) { + buffered.dead(); + } + } + return Mono.empty(); + }), + 8) + .then(Reactors.ALWAYS_FALSE); + } + + public Mono save0(String metric, Collection> data) { + + return database + .getMetadata() + .getTableOrViewReactive(metric, false) + .cast(RDBTableMetadata.class) + .switchIfEmpty(Mono.error(() -> new BusinessException.NoStackTrace("metric [" + metric + "] not found"))) + .flatMap(table -> database + .dml() + .upsert(table) + .values(data instanceof List ? ((List>) data) : new ArrayList<>(data)) + .execute() + .reactive() + .then()) + .contextWrite(ctx -> ctx.put(Logger.class, log)); + } + + private boolean needRetry(Throwable err) { + return ErrorUtils.hasException( + err, + R2dbcException.class, + IOException.class, + IllegalStateException.class, + RejectedExecutionException.class, + TimeoutException.class, + DataAccessResourceFailureException.class, + CannotCreateTransactionException.class, + QueryTimeoutException.class); + } + + + @Override + public void run(String... args) { + start(); + SpringApplication + .getShutdownHandlers() + .add(buffer::dispose); + } + + @Getter + @NoArgsConstructor + @AllArgsConstructor + public static class Buffer implements Externalizable { + private String metric; + private Map data; + + @Override + public void writeExternal(ObjectOutput out) throws IOException { + SerializeUtils.writeNullableUTF(metric, out); + SerializeUtils.writeKeyValue(data, out); + } + + @Override + public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { + metric = SerializeUtils.readNullableUTF(in); + data = SerializeUtils.readMap(in, Maps::newHashMapWithExpectedSize); + } + } +} diff --git a/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/impl/DefaultTimescaleDBOperations.java b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/impl/DefaultTimescaleDBOperations.java new file mode 100644 index 00000000..93fc0415 --- /dev/null +++ b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/impl/DefaultTimescaleDBOperations.java @@ -0,0 +1,109 @@ +package org.jetlinks.community.timescaledb.impl; + +import com.google.common.collect.Maps; +import lombok.RequiredArgsConstructor; +import org.hswebframework.ezorm.rdb.executor.reactive.ReactiveSqlExecutor; +import org.hswebframework.ezorm.rdb.executor.reactive.ReactiveSyncSqlExecutor; +import org.hswebframework.ezorm.rdb.metadata.RDBDatabaseMetadata; +import org.hswebframework.ezorm.rdb.metadata.RDBSchemaMetadata; +import org.hswebframework.ezorm.rdb.metadata.dialect.Dialect; +import org.hswebframework.ezorm.rdb.operator.DatabaseOperator; +import org.hswebframework.ezorm.rdb.operator.DefaultDatabaseOperator; +import org.jetlinks.community.datasource.rdb.RDBDataSource; +import org.jetlinks.community.datasource.rdb.RDBDataSourceProperties; +import org.jetlinks.community.datasource.rdb.RDBDataSourceProvider; +import org.jetlinks.community.timescaledb.TimescaleDBDataWriter; +import org.jetlinks.community.timescaledb.TimescaleDBOperations; +import org.jetlinks.community.timescaledb.TimescaleDBProperties; +import org.jetlinks.community.timescaledb.metadata.TimescaleDBDialectProvider; +import org.springframework.beans.BeansException; +import org.springframework.boot.CommandLineRunner; +import org.springframework.boot.SpringApplication; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import reactor.core.Disposable; +import reactor.core.Disposables; + +import javax.annotation.Nonnull; +import java.util.Map; + +@RequiredArgsConstructor +public class DefaultTimescaleDBOperations implements TimescaleDBOperations, ApplicationContextAware, CommandLineRunner { + + private final TimescaleDBProperties properties; + private final Disposable.Composite disposable = Disposables.composite(); + + private ApplicationContext context; + private DatabaseOperator database; + private DefaultTimescaleDBDataWriter writer; + + + public void shutdown() { + disposable.dispose(); + } + + public void init() { + if (properties.isSharedSpring() && context != null) { + //使用spring共享数据源 + ReactiveSqlExecutor sqlExecutor = context.getBean(ReactiveSqlExecutor.class); + RDBDatabaseMetadata database = new RDBDatabaseMetadata(Dialect.POSTGRES); + database.addFeature(sqlExecutor); + database.addFeature(ReactiveSyncSqlExecutor.of(sqlExecutor)); + + RDBSchemaMetadata schema = TimescaleDBDialectProvider.GLOBAL.createSchema(properties.getSchema()); + database.addSchema(schema); + database.setCurrentSchema(schema); + this.database = DefaultDatabaseOperator.of(database); + } else { + if (properties.getR2dbc() == null) { + throw new IllegalArgumentException("timescaledb.r2dbc must not be null"); + } + RDBDataSourceProperties datasource = new RDBDataSourceProperties(); + datasource.setType(RDBDataSourceProperties.Type.r2dbc); + datasource.setSchema(properties.getSchema()); + datasource.setUsername(properties.getR2dbc().getUsername()); + datasource.setPassword(properties.getR2dbc().getPassword()); + datasource.setUrl(properties.getR2dbc().getUrl()); + datasource.setDialect(TimescaleDBDialectProvider.NAME); + + Map others = Maps.newHashMap(); + others.put("properties", properties.getR2dbc().getProperties()); + others.put("pool", properties.getR2dbc().getPool()); + + datasource.setOthers(others); + + RDBDataSource dataSource = RDBDataSourceProvider + .create("TimescaleDB", datasource); + disposable.add(dataSource); + database = dataSource.operator(); + } + writer = new DefaultTimescaleDBDataWriter(database, properties.getWriteBuffer()); + writer.init(); + disposable.add(writer::stop); + } + + @Override + public DatabaseOperator database() { + return database; + } + + @Override + public TimescaleDBDataWriter writer() { + return writer; + } + + @Override + public void setApplicationContext(@Nonnull ApplicationContext context) throws BeansException { + this.context = context; + } + + @Override + public void run(String... args) { + if (writer != null) { + SpringApplication + .getShutdownHandlers() + .add(writer::shutdown); + writer.start(); + } + } +} diff --git a/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/metadata/CreateHypertable.java b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/metadata/CreateHypertable.java new file mode 100644 index 00000000..e3b2a423 --- /dev/null +++ b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/metadata/CreateHypertable.java @@ -0,0 +1,34 @@ +package org.jetlinks.community.timescaledb.metadata; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.hswebframework.ezorm.core.FeatureId; +import org.hswebframework.ezorm.core.FeatureType; +import org.hswebframework.ezorm.core.meta.Feature; +import org.jetlinks.community.Interval; + +@Getter +@AllArgsConstructor +public class CreateHypertable implements Feature, FeatureType { + + public static final FeatureId ID = FeatureId.of("CreateHypertable"); + + private final String column; + + private final Interval chunkTimeInterval; + + @Override + public String getId() { + return ID.getId(); + } + + @Override + public String getName() { + return "CreateHypertable"; + } + + @Override + public FeatureType getType() { + return this; + } +} diff --git a/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/metadata/CreateRetentionPolicy.java b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/metadata/CreateRetentionPolicy.java new file mode 100644 index 00000000..9e503066 --- /dev/null +++ b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/metadata/CreateRetentionPolicy.java @@ -0,0 +1,32 @@ +package org.jetlinks.community.timescaledb.metadata; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.hswebframework.ezorm.core.FeatureId; +import org.hswebframework.ezorm.core.FeatureType; +import org.hswebframework.ezorm.core.meta.Feature; +import org.jetlinks.community.Interval; + +@Getter +@AllArgsConstructor +public class CreateRetentionPolicy implements Feature, FeatureType { + + public static final FeatureId ID = FeatureId.of("CreateRetentionPolicy"); + + private final Interval interval; + + @Override + public String getId() { + return ID.getId(); + } + + @Override + public String getName() { + return "CreateRetentionPolicy"; + } + + @Override + public FeatureType getType() { + return this; + } +} diff --git a/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/metadata/JsonbValueCodec.java b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/metadata/JsonbValueCodec.java new file mode 100644 index 00000000..8b4f0988 --- /dev/null +++ b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/metadata/JsonbValueCodec.java @@ -0,0 +1,50 @@ +package org.jetlinks.community.timescaledb.metadata; + +import io.r2dbc.postgresql.codec.Json; +import org.hswebframework.ezorm.rdb.codec.JsonValueCodec; +import org.hswebframework.ezorm.rdb.operator.builder.fragments.NativeSql; +import org.jetlinks.community.utils.ObjectMappers; + +import java.util.ArrayList; +import java.util.LinkedHashMap; + +public class JsonbValueCodec extends JsonValueCodec { + + public JsonbValueCodec(boolean array) { + super(array ? ArrayList.class : LinkedHashMap.class, + array ? ObjectMappers.JSON_MAPPER + .getTypeFactory() + .constructCollectionType(ArrayList.class, Object.class) : + ObjectMappers.JSON_MAPPER + .getTypeFactory() + .constructMapType(LinkedHashMap.class, Object.class, Object.class)); + } + + @Override + public Object encode(Object value) { + Object obj = super.encode(value); + if (obj == null) { + return encodeNull(); + } + return NativeSql.of("?::jsonb", obj); + } + + @Override + public Object encodeNull() { + return NativeSql.of("null::jsonb"); + } + + @Override + public Object decode(Object data) { + if (data instanceof io.r2dbc.postgresql.codec.Json) { + byte[] arr = ((Json) data).asArray(); + + if (arr[0] == '[') { + return ObjectMappers.parseJsonArray(arr, Object.class); + } + return ObjectMappers.parseJson(arr, Object.class); + + } + return super.decode(data); + } +} \ No newline at end of file diff --git a/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/metadata/TimescaleDBAlterTableSqlBuilder.java b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/metadata/TimescaleDBAlterTableSqlBuilder.java new file mode 100644 index 00000000..6b309006 --- /dev/null +++ b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/metadata/TimescaleDBAlterTableSqlBuilder.java @@ -0,0 +1,23 @@ +package org.jetlinks.community.timescaledb.metadata; + +import org.hswebframework.ezorm.rdb.executor.DefaultBatchSqlRequest; +import org.hswebframework.ezorm.rdb.executor.SqlRequest; +import org.hswebframework.ezorm.rdb.executor.SqlRequests; +import org.hswebframework.ezorm.rdb.metadata.RDBIndexMetadata; +import org.hswebframework.ezorm.rdb.metadata.RDBTableMetadata; +import org.hswebframework.ezorm.rdb.operator.builder.fragments.ddl.CommonCreateTableSqlBuilder; +import org.hswebframework.ezorm.rdb.supports.postgres.PostgresqlAlterTableSqlBuilder; + +public class TimescaleDBAlterTableSqlBuilder extends PostgresqlAlterTableSqlBuilder { + + + @Override + protected void appendDropIndexSql(DefaultBatchSqlRequest batch, RDBTableMetadata table, RDBIndexMetadata index) { + + } + + @Override + protected void appendAddIndexSql(DefaultBatchSqlRequest batch, RDBTableMetadata table, RDBIndexMetadata index) { + + } +} diff --git a/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/metadata/TimescaleDBCreateTableSqlBuilder.java b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/metadata/TimescaleDBCreateTableSqlBuilder.java new file mode 100644 index 00000000..8317f7cd --- /dev/null +++ b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/metadata/TimescaleDBCreateTableSqlBuilder.java @@ -0,0 +1,47 @@ +package org.jetlinks.community.timescaledb.metadata; + +import org.hswebframework.ezorm.rdb.executor.DefaultBatchSqlRequest; +import org.hswebframework.ezorm.rdb.executor.SqlRequest; +import org.hswebframework.ezorm.rdb.executor.SqlRequests; +import org.hswebframework.ezorm.rdb.metadata.RDBTableMetadata; +import org.hswebframework.ezorm.rdb.operator.builder.fragments.ddl.CommonCreateTableSqlBuilder; + +public class TimescaleDBCreateTableSqlBuilder extends CommonCreateTableSqlBuilder { + + @Override + public SqlRequest build(RDBTableMetadata table) { + DefaultBatchSqlRequest sqlRequest = (DefaultBatchSqlRequest) super.build(table); + + table.getFeature(CreateHypertable.ID) + .ifPresent(createHypertable -> sqlRequest.addBatch(createCreateHypertableSQL(table, createHypertable))); + + table.getFeature(CreateRetentionPolicy.ID) + .ifPresent(feature -> sqlRequest.addBatch(createCreateRetentionPolicySQL(table, feature))); + + return sqlRequest; + } + + + private SqlRequest createCreateRetentionPolicySQL(RDBTableMetadata table, CreateRetentionPolicy createHypertable) { + + String interval = createHypertable.getInterval().getNumber().intValue() + " " + + createHypertable.getInterval().getUnit().name().toLowerCase(); + + return SqlRequests.of( + "SELECT add_retention_policy( ? , INTERVAL '" + interval + "')", + table.getFullName() + ); + } + + private SqlRequest createCreateHypertableSQL(RDBTableMetadata table, CreateHypertable createHypertable) { + + String interval = createHypertable.getChunkTimeInterval().getNumber().intValue() + " " + + createHypertable.getChunkTimeInterval().getUnit().name().toLowerCase(); + + return SqlRequests.of( + "SELECT create_hypertable( ? , ? , chunk_time_interval => INTERVAL '" + interval + "')", + table.getFullName(), + table.getColumnNow(createHypertable.getColumn()).getName() + ); + } +} diff --git a/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/metadata/TimescaleDBDialect.java b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/metadata/TimescaleDBDialect.java new file mode 100644 index 00000000..932ed010 --- /dev/null +++ b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/metadata/TimescaleDBDialect.java @@ -0,0 +1,10 @@ +package org.jetlinks.community.timescaledb.metadata; + +import org.hswebframework.ezorm.rdb.supports.postgres.PostgresqlDialect; + +public class TimescaleDBDialect extends PostgresqlDialect { + + public TimescaleDBDialect(){ + + } +} diff --git a/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/metadata/TimescaleDBDialectProvider.java b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/metadata/TimescaleDBDialectProvider.java new file mode 100644 index 00000000..59f21da6 --- /dev/null +++ b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/metadata/TimescaleDBDialectProvider.java @@ -0,0 +1,53 @@ +package org.jetlinks.community.timescaledb.metadata; + +import org.hswebframework.ezorm.rdb.codec.DateTimeCodec; +import org.hswebframework.ezorm.rdb.metadata.DefaultValueCodecFactory; +import org.hswebframework.ezorm.rdb.metadata.RDBSchemaMetadata; +import org.hswebframework.ezorm.rdb.metadata.dialect.Dialect; +import org.hswebframework.ezorm.rdb.supports.postgres.PostgresqlSchemaMetadata; +import org.hswebframework.web.crud.configuration.DialectProvider; + +import java.sql.JDBCType; +import java.util.Date; + +public class TimescaleDBDialectProvider implements DialectProvider { + + public static final TimescaleDBDialectProvider GLOBAL = new TimescaleDBDialectProvider(); + + public static final String NAME = "timescaledb"; + + @Override + public String name() { + return "timescaledb"; + } + + @Override + public Dialect getDialect() { + return Dialect.POSTGRES; + } + + @Override + public String getBindSymbol() { + return "$"; + } + + @Override + public RDBSchemaMetadata createSchema(String name) { + PostgresqlSchemaMetadata schema = new PostgresqlSchemaMetadata(name); + schema.addFeature(new TimescaleDBCreateTableSqlBuilder()); + schema.addFeature(new TimescaleDBAlterTableSqlBuilder()); + DefaultValueCodecFactory codecFactory = new DefaultValueCodecFactory(); + codecFactory + .register(col -> "jsonb".equals(col.getDataType()) || + "json".equals(col.getDataType()), + col -> new JsonbValueCodec(true)); + + codecFactory + .register(col -> col.getType().getSqlType() == JDBCType.TIMESTAMP + || col.getType().getSqlType() == JDBCType.TIMESTAMP_WITH_TIMEZONE, + col -> new DateTimeCodec("yyyy-MM-dd HH:mm:ss.SSS", Date.class)); + + schema.addFeature(codecFactory); + return schema; + } +} diff --git a/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/thing/TimescaleDBColumnModeDDLOperations.java b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/thing/TimescaleDBColumnModeDDLOperations.java new file mode 100644 index 00000000..b65c9301 --- /dev/null +++ b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/thing/TimescaleDBColumnModeDDLOperations.java @@ -0,0 +1,130 @@ +package org.jetlinks.community.timescaledb.thing; + +import lombok.extern.slf4j.Slf4j; +import org.hswebframework.ezorm.rdb.codec.DateTimeCodec; +import org.hswebframework.ezorm.rdb.metadata.RDBIndexMetadata; +import org.hswebframework.ezorm.rdb.metadata.RDBSchemaMetadata; +import org.hswebframework.ezorm.rdb.metadata.RDBTableMetadata; +import org.hswebframework.ezorm.rdb.operator.DatabaseOperator; +import org.hswebframework.ezorm.rdb.operator.ddl.TableBuilder; +import org.jetlinks.core.metadata.PropertyMetadata; +import org.jetlinks.community.Interval; +import org.jetlinks.community.things.data.ThingsDataConstants; +import org.jetlinks.community.things.data.operations.ColumnModeDDLOperationsBase; +import org.jetlinks.community.things.data.operations.DataSettings; +import org.jetlinks.community.things.data.operations.MetricBuilder; +import org.jetlinks.community.things.utils.ThingsDatabaseUtils; +import org.jetlinks.community.timescaledb.TimescaleDBUtils; +import org.jetlinks.community.timescaledb.metadata.CreateHypertable; +import org.slf4j.Logger; +import reactor.core.publisher.Mono; + +import java.sql.JDBCType; +import java.util.*; + +@Slf4j +public class TimescaleDBColumnModeDDLOperations extends ColumnModeDDLOperationsBase { + + private final DatabaseOperator database; + + private final TimescaleDBThingsDataProperties properties; + + public TimescaleDBColumnModeDDLOperations(String thingType, + String templateId, + String thingId, + DataSettings settings, + MetricBuilder metricBuilder, + DatabaseOperator database, + TimescaleDBThingsDataProperties properties) { + super(thingType, templateId, thingId, settings, metricBuilder); + this.database = database; + this.properties = properties; + } + + protected Mono register0(MetricType metricType, + String metric, + List properties, + boolean ddl) { + metric = TimescaleDBUtils.getTableName(metric); + RDBSchemaMetadata schema = database.getMetadata().getCurrentSchema(); + RDBTableMetadata table = schema.newTable(metric); + TableBuilder builder = database + .ddl() + .createOrAlter(table); + + List partitions = new ArrayList<>(); + partitions.add(ThingsDataConstants.COLUMN_THING_ID); + for (PropertyMetadata property : properties) { + builder + .addColumn(property.getId()) + .custom(column -> { + ThingsDatabaseUtils.convertColumn(property, column); + if (Objects + .equals(ThingsDataConstants.COLUMN_TIMESTAMP, column.getName())) { + column.setNotNull(true); + column.setJdbcType(JDBCType.TIMESTAMP, Date.class); + column.setValueCodec(new DateTimeCodec("yyyy-MM-dd HH:mm:ss.SSS", Date.class)); + } + TimescaleDBUtils.customColumn(property,column); + }) + .commit(); + } + + if (metricType == MetricType.properties) { + //索引 + builder + .index() + .name("idx_" + metric + "_prop_id") + .column(metricBuilder.getThingIdProperty()) + .column(ThingsDataConstants.COLUMN_TIMESTAMP, RDBIndexMetadata.IndexSort.desc) + .unique() + .commit(); + } else if (metricType == MetricType.event && settings.getEvent().eventIsAllInOne()) { + partitions.add(ThingsDataConstants.COLUMN_EVENT_ID); + //索引 + builder + .index() + .name("idx_" + metric + "_event_id") + .column(metricBuilder.getThingIdProperty()) + .column(ThingsDataConstants.COLUMN_EVENT_ID) + .column(ThingsDataConstants.COLUMN_TIMESTAMP, RDBIndexMetadata.IndexSort.desc) + .unique() + .commit(); + } else if (metricType == MetricType.log) { + //索引 + builder + .index() + .name("idx_" + metric + "_log_id") + .column(metricBuilder.getThingIdProperty()) + .column(ThingsDataConstants.COLUMN_LOG_TYPE) + .commit(); + } + + table.addFeature(new CreateHypertable(ThingsDataConstants.COLUMN_TIMESTAMP, this.properties.getChunkTimeInterval())); + + if (ddl) { + return builder + .commit() + .reactive() + .then() + .contextWrite(ctx -> ctx.put(Logger.class, log)); + } + return schema + .getTableReactive(metric, true) + .doOnNext(oldTable -> oldTable.replace(table)) + .switchIfEmpty(Mono.fromRunnable(() -> schema.addTable(table))) + .then() + .contextWrite(ctx -> ctx.put(Logger.class, log)); + } + + @Override + protected Mono register(MetricType metricType, String metric, List properties) { + + return register0(metricType, metric, properties, true); + } + + @Override + protected Mono reload(MetricType metricType, String metric, List properties) { + return register0(metricType, metric, properties, false); + } +} diff --git a/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/thing/TimescaleDBColumnModeQueryOperations.java b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/thing/TimescaleDBColumnModeQueryOperations.java new file mode 100644 index 00000000..8ab0750e --- /dev/null +++ b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/thing/TimescaleDBColumnModeQueryOperations.java @@ -0,0 +1,192 @@ +package org.jetlinks.community.timescaledb.thing; + +import com.google.common.collect.Maps; +import lombok.extern.slf4j.Slf4j; +import org.hswebframework.ezorm.core.Conditional; +import org.hswebframework.ezorm.core.dsl.Query; +import org.hswebframework.ezorm.rdb.executor.wrapper.ResultWrappers; +import org.hswebframework.ezorm.rdb.mapping.defaults.record.Record; +import org.hswebframework.ezorm.rdb.operator.DatabaseOperator; +import org.hswebframework.ezorm.rdb.operator.dml.QueryOperator; +import org.hswebframework.ezorm.rdb.operator.dml.query.NativeSelectColumn; +import org.hswebframework.ezorm.rdb.operator.dml.query.SelectColumn; +import org.hswebframework.web.api.crud.entity.PagerResult; +import org.hswebframework.web.api.crud.entity.QueryParamEntity; +import org.hswebframework.web.crud.query.QueryHelper; +import org.jetlinks.core.metadata.EventMetadata; +import org.jetlinks.core.things.ThingsRegistry; +import org.jetlinks.community.things.data.AggregationRequest; +import org.jetlinks.community.things.data.PropertyAggregation; +import org.jetlinks.community.things.data.ThingsDataConstants; +import org.jetlinks.community.things.data.ThingsDataUtils; +import org.jetlinks.community.things.data.operations.ColumnModeQueryOperationsBase; +import org.jetlinks.community.things.data.operations.DataSettings; +import org.jetlinks.community.things.data.operations.MetricBuilder; +import org.jetlinks.community.things.data.operations.RowModeQueryOperationsBase; +import org.jetlinks.community.timescaledb.TimescaleDBUtils; +import org.jetlinks.community.timeseries.TimeSeriesData; +import org.jetlinks.community.timeseries.query.Aggregation; +import org.jetlinks.community.timeseries.query.AggregationData; +import org.jetlinks.reactor.ql.utils.CastUtils; +import org.slf4j.Logger; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.util.*; +import java.util.function.Function; + +import static org.jetlinks.community.timescaledb.TimescaleDBUtils.createTimeGroupColumn; + +@Slf4j +public class TimescaleDBColumnModeQueryOperations extends ColumnModeQueryOperationsBase { + private final DatabaseOperator database; + + public TimescaleDBColumnModeQueryOperations(String thingType, + String thingTemplateId, + String thingId, + MetricBuilder metricBuilder, + DataSettings settings, + ThingsRegistry registry, + DatabaseOperator database) { + super(thingType, thingTemplateId, thingId, metricBuilder, settings, registry); + this.database = database; + } + + @Override + protected Flux doQuery(String metric, Query query) { + metric = TimescaleDBUtils.getTableName(metric); + return query + .execute( + database + .dml() + .createReactiveRepository(metric) + .createQuery()::setParam + ) + .fetch() + .map(this::convertToTimeSeriesData) + .contextWrite(ctx -> ctx.put(Logger.class, log)); + } + + @Override + protected Mono> doQueryPage(String metric, + Query query, + Function mapper) { + return QueryHelper + .queryPager( + query.getParam(), + () -> database + .dml() + .createReactiveRepository(TimescaleDBUtils.getTableName(metric)) + .createQuery(), + record -> mapper.apply(convertToTimeSeriesData(record)) + ) + .contextWrite(ctx -> ctx.put(Logger.class, log)); + } + + static final String timestampAlias = "_ts"; + + @Override + protected Flux doAggregation(String metric, + AggregationRequest request, + AggregationContext context) { + return doAggregation0(database, metric, request, context); + } + + + static Flux doAggregation0(DatabaseOperator database, + String metric, + AggregationRequest request, + AggregationContext context) { + metric = TimescaleDBUtils.getTableName(metric); + QueryParamEntity filter = request.getFilter(); + filter.setSorts(new ArrayList<>()); + filter.setPaging(false); + QueryOperator query = database.dml().query(metric); + + //按时间分组 + if (request.getInterval() != null) { + NativeSelectColumn column = createTimeGroupColumn( + request.getFrom().getTime(), + request.getInterval() + ); + query.groupBy(column); + query.select(column); + column.setAlias(timestampAlias); + } + + for (PropertyAggregation property : context.getProperties()) { + SelectColumn column = new SelectColumn(); + column.setColumn(property.getProperty()); + column.setAlias(property.getAlias()); + TimescaleDBUtils.applyAggColumn(property.getAgg(), column); + + query.select(column); + } + + query.where(cdt -> { + cdt.each(filter.getTerms(), Conditional::accept); + cdt.between(ThingsDataConstants.COLUMN_TIMESTAMP, request.getFrom(), request.getTo()); + }); + + NavigableMap> + prepares = ThingsDataUtils.prepareAggregationData(request, context.getProperties()); + + return query + .fetch(ResultWrappers.map()) + .reactive() + .map(val -> Maps.filterValues(val, Objects::nonNull)) + .map(AggregationData::of) + .groupBy(data -> data.getLong(timestampAlias).orElse(0L), Integer.MAX_VALUE) + .flatMap(group -> { + long time = group.key(); + Map prepare = ThingsDataUtils.findAggregationData(time, prepares); + if (prepare == null) { + return Mono.empty(); + } + return group + .doOnNext(data -> { + for (PropertyAggregation property : context.getProperties()) { + String alias = property.getAlias(); + data.get(alias) + .ifPresent(val -> prepare.put(alias, val)); + } + }); + }) + .thenMany(Flux.fromIterable(prepares.values())) + .map(AggregationData::of) + .take((long) request.getLimit() * context.getProperties().length) + .contextWrite(ctx -> ctx.put(Logger.class, log)); + + } + + + private TimeSeriesData convertToTimeSeriesData(Record record) { + return TimeSeriesData.of( + record.get(ThingsDataConstants.COLUMN_TIMESTAMP) + .map(val -> CastUtils.castNumber(val).longValue()) + .orElseGet(System::currentTimeMillis), + record + ); + } + + + protected String createAggFunction(Aggregation aggregation) { + switch (aggregation) { + case COUNT: + return "count(1)"; + case DISTINCT_COUNT: + return "count(distinct \"numberValue\")"; + case AVG: + return "avg(\"numberValue\")"; + case MAX: + return "max(\"numberValue\")"; + case MIN: + return "min(\"numberValue\")"; + case SUM: + return "sum(\"numberValue\")"; +// case STDDEV: +// return "stddev(\"numberValue\")"; + } + throw new UnsupportedOperationException("不支持的聚合函数:" + aggregation); + } +} diff --git a/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/thing/TimescaleDBColumnModeSaveOperations.java b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/thing/TimescaleDBColumnModeSaveOperations.java new file mode 100644 index 00000000..e5920d7e --- /dev/null +++ b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/thing/TimescaleDBColumnModeSaveOperations.java @@ -0,0 +1,35 @@ +package org.jetlinks.community.timescaledb.thing; + +import org.jetlinks.core.things.ThingsRegistry; +import org.jetlinks.community.things.data.operations.ColumnModeSaveOperationsBase; +import org.jetlinks.community.things.data.operations.DataSettings; +import org.jetlinks.community.things.data.operations.MetricBuilder; +import org.jetlinks.community.things.data.operations.RowModeSaveOperationsBase; +import org.jetlinks.community.timescaledb.TimescaleDBDataWriter; +import org.jetlinks.community.timescaledb.TimescaleDBUtils; +import org.jetlinks.community.timeseries.TimeSeriesData; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +public class TimescaleDBColumnModeSaveOperations extends ColumnModeSaveOperationsBase { + + private final TimescaleDBDataWriter writer; + + public TimescaleDBColumnModeSaveOperations(ThingsRegistry registry, + MetricBuilder metricBuilder, + DataSettings settings, + TimescaleDBDataWriter writer) { + super(registry, metricBuilder, settings); + this.writer = writer; + } + + @Override + protected Mono doSave(String metric, TimeSeriesData data) { + return writer.save(TimescaleDBUtils.getTableName(metric), data.getData()); + } + + @Override + protected Mono doSave(String metric, Flux data) { + return writer.save(TimescaleDBUtils.getTableName(metric), data.map(TimeSeriesData::getData)); + } +} diff --git a/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/thing/TimescaleDBColumnModeStrategy.java b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/thing/TimescaleDBColumnModeStrategy.java new file mode 100644 index 00000000..4c81b873 --- /dev/null +++ b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/thing/TimescaleDBColumnModeStrategy.java @@ -0,0 +1,80 @@ +package org.jetlinks.community.timescaledb.thing; + +import org.jetlinks.community.things.data.TableSafeMetricBuilder; +import org.jetlinks.core.things.ThingsRegistry; +import org.jetlinks.community.things.data.AbstractThingDataRepositoryStrategy; +import org.jetlinks.community.things.data.operations.DDLOperations; +import org.jetlinks.community.things.data.operations.QueryOperations; +import org.jetlinks.community.things.data.operations.SaveOperations; +import org.jetlinks.community.timescaledb.TimescaleDBOperations; + +public class TimescaleDBColumnModeStrategy extends AbstractThingDataRepositoryStrategy { + + private final ThingsRegistry registry; + private final TimescaleDBOperations operations; + private final TimescaleDBThingsDataProperties properties; + + public TimescaleDBColumnModeStrategy(ThingsRegistry registry, + TimescaleDBOperations operations) { + this(registry, operations, new TimescaleDBThingsDataProperties()); + } + + public TimescaleDBColumnModeStrategy(ThingsRegistry registry, + TimescaleDBOperations operations, + TimescaleDBThingsDataProperties properties) { + this.registry = registry; + this.operations = operations; + this.properties = properties; + } + + @Override + public String getId() { + return "timescaledb-column"; + } + + @Override + public String getName() { + return "TimescaleDB-多列模式"; + } + + @Override + public SaveOperations createOpsForSave(OperationsContext context) { + return new TimescaleDBColumnModeSaveOperations( + registry, + TableSafeMetricBuilder.of(context.getMetricBuilder()), + context.getSettings(), + operations.writer()); + } + + @Override + protected QueryOperations createForQuery(String thingType, String templateId, String thingId, OperationsContext context) { + return new TimescaleDBColumnModeQueryOperations( + thingType, + templateId, + thingId, + TableSafeMetricBuilder.of(context.getMetricBuilder()), + context.getSettings(), + registry, + operations.database()); + } + + @Override + protected DDLOperations createForDDL(String thingType, + String templateId, + String thingId, + OperationsContext context) { + return new TimescaleDBColumnModeDDLOperations( + thingType, + templateId, + thingId, + context.getSettings(), + TableSafeMetricBuilder.of(context.getMetricBuilder()), + operations.database(), + properties); + } + + @Override + public int getOrder() { + return 10070; + } +} diff --git a/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/thing/TimescaleDBRowModeDDLOperations.java b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/thing/TimescaleDBRowModeDDLOperations.java new file mode 100644 index 00000000..b784159b --- /dev/null +++ b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/thing/TimescaleDBRowModeDDLOperations.java @@ -0,0 +1,148 @@ +package org.jetlinks.community.timescaledb.thing; + +import com.google.common.collect.Sets; +import lombok.extern.slf4j.Slf4j; +import org.hswebframework.ezorm.rdb.codec.DateTimeCodec; +import org.hswebframework.ezorm.rdb.metadata.RDBIndexMetadata; +import org.hswebframework.ezorm.rdb.metadata.RDBSchemaMetadata; +import org.hswebframework.ezorm.rdb.metadata.RDBTableMetadata; +import org.hswebframework.ezorm.rdb.operator.DatabaseOperator; +import org.hswebframework.ezorm.rdb.operator.ddl.TableBuilder; +import org.jetlinks.core.metadata.PropertyMetadata; +import org.jetlinks.community.things.data.ThingsDataConstants; +import org.jetlinks.community.things.data.operations.DataSettings; +import org.jetlinks.community.things.data.operations.MetricBuilder; +import org.jetlinks.community.things.data.operations.RowModeDDLOperationsBase; +import org.jetlinks.community.things.utils.ThingsDatabaseUtils; +import org.jetlinks.community.timescaledb.TimescaleDBUtils; +import org.jetlinks.community.timescaledb.metadata.CreateHypertable; +import org.jetlinks.community.timescaledb.metadata.CreateRetentionPolicy; +import org.slf4j.Logger; +import reactor.core.publisher.Mono; + +import java.sql.JDBCType; +import java.util.*; + +@Slf4j +public class TimescaleDBRowModeDDLOperations extends RowModeDDLOperationsBase { + + private final DatabaseOperator database; + private final TimescaleDBThingsDataProperties properties; + + static Set ignoreColumn = Sets.newHashSet( + ThingsDataConstants.COLUMN_ID, + ThingsDataConstants.COLUMN_MESSAGE_ID, + ThingsDataConstants.COLUMN_PROPERTY_GEO_VALUE, + ThingsDataConstants.COLUMN_PROPERTY_OBJECT_VALUE, + ThingsDataConstants.COLUMN_PROPERTY_ARRAY_VALUE + ); + + public TimescaleDBRowModeDDLOperations(String thingType, + String templateId, + String thingId, + DataSettings settings, + MetricBuilder metricBuilder, + DatabaseOperator database, + TimescaleDBThingsDataProperties properties) { + super(thingType, templateId, thingId, settings, metricBuilder); + this.database = database; + this.properties = properties; + } + + protected Mono register0(MetricType metricType, + String metric, + List properties, + boolean ddl) { + metric = TimescaleDBUtils.getTableName(metric); + RDBSchemaMetadata schema = database.getMetadata().getCurrentSchema(); + RDBTableMetadata table = schema.newTable(metric); + TableBuilder builder = database + .ddl() + .createOrAlter(table); + + + List partitions = new ArrayList<>(); + partitions.add(ThingsDataConstants.COLUMN_THING_ID); + for (PropertyMetadata property : properties) { + if (ignoreColumn.contains(property.getId())) { + continue; + } + builder + .addColumn(property.getId()) + .custom(column -> { + ThingsDatabaseUtils.convertColumn(property, column); + if (Objects + .equals(ThingsDataConstants.COLUMN_TIMESTAMP, column.getName())) { + column.setNotNull(true); + column.setJdbcType(JDBCType.TIMESTAMP, Date.class); + column.setValueCodec(new DateTimeCodec("yyyy-MM-dd HH:mm:ss.SSS", Date.class)); + } + TimescaleDBUtils.customColumn(property,column); + }) + .commit(); + } + + if (metricType == MetricType.properties) { + //索引 + builder + .index() + .name("idx_" + metric + "_prop_id") + .column(metricBuilder.getThingIdProperty()) + .column(ThingsDataConstants.COLUMN_PROPERTY_ID) + .column(ThingsDataConstants.COLUMN_TIMESTAMP, RDBIndexMetadata.IndexSort.desc) + .unique() + .commit(); + } else if (metricType == MetricType.event && settings.getEvent().eventIsAllInOne()) { + partitions.add(ThingsDataConstants.COLUMN_EVENT_ID); + //索引 + builder + .index() + .name("idx_" + metric + "_event_id") + .column(metricBuilder.getThingIdProperty()) + .column(ThingsDataConstants.COLUMN_EVENT_ID) + .column(ThingsDataConstants.COLUMN_TIMESTAMP, RDBIndexMetadata.IndexSort.desc) + .unique() + .commit(); + } else if (metricType == MetricType.log) { + //索引 + builder + .index() + .name("idx_" + metric + "_log_id") + .column(metricBuilder.getThingIdProperty()) + .column(ThingsDataConstants.COLUMN_TIMESTAMP, RDBIndexMetadata.IndexSort.desc) + .column(ThingsDataConstants.COLUMN_LOG_TYPE) + .commit(); + } + + table.addFeature(new CreateHypertable(ThingsDataConstants.COLUMN_TIMESTAMP, this.properties.getChunkTimeInterval())); + + if (this.properties.getRetentionPolicy() != null) { + table.addFeature(new CreateRetentionPolicy(this.properties.getRetentionPolicy())); + } + + if (ddl) { + return builder + .commit() + .reactive() + .then() + .contextWrite(ctx -> ctx.put(Logger.class, log)); + } + return schema + .getTableReactive(metric, true) + .doOnNext(oldTable -> oldTable.replace(table)) + .switchIfEmpty(Mono.fromRunnable(() -> schema.addTable(table))) + .then() + .contextWrite(ctx -> ctx.put(Logger.class, log)); + } + + @Override + protected Mono register(MetricType metricType, String metric, List properties) { + + return register0(metricType, metric, properties, true); + } + + @Override + protected Mono reload(MetricType metricType, String metric, List properties) { + return register0(metricType, metric, properties, false); + } +} diff --git a/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/thing/TimescaleDBRowModeQueryOperations.java b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/thing/TimescaleDBRowModeQueryOperations.java new file mode 100644 index 00000000..f57f1695 --- /dev/null +++ b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/thing/TimescaleDBRowModeQueryOperations.java @@ -0,0 +1,203 @@ +package org.jetlinks.community.timescaledb.thing; + +import com.google.common.collect.Maps; +import lombok.extern.slf4j.Slf4j; +import org.hswebframework.ezorm.core.Conditional; +import org.hswebframework.ezorm.core.dsl.Query; +import org.hswebframework.ezorm.rdb.executor.wrapper.ResultWrappers; +import org.hswebframework.ezorm.rdb.mapping.defaults.record.Record; +import org.hswebframework.ezorm.rdb.operator.DatabaseOperator; +import org.hswebframework.ezorm.rdb.operator.dml.QueryOperator; +import org.hswebframework.ezorm.rdb.operator.dml.query.NativeSelectColumn; +import org.hswebframework.ezorm.rdb.operator.dml.query.SelectColumn; +import org.hswebframework.web.api.crud.entity.PagerResult; +import org.hswebframework.web.api.crud.entity.QueryParamEntity; +import org.hswebframework.web.crud.query.QueryHelper; +import org.jetlinks.core.metadata.EventMetadata; +import org.jetlinks.core.things.ThingsRegistry; +import org.jetlinks.community.things.data.AggregationRequest; +import org.jetlinks.community.things.data.PropertyAggregation; +import org.jetlinks.community.things.data.ThingsDataConstants; +import org.jetlinks.community.things.data.ThingsDataUtils; +import org.jetlinks.community.things.data.operations.DataSettings; +import org.jetlinks.community.things.data.operations.MetricBuilder; +import org.jetlinks.community.things.data.operations.RowModeQueryOperationsBase; +import org.jetlinks.community.timescaledb.TimescaleDBUtils; +import org.jetlinks.community.timeseries.TimeSeriesData; +import org.jetlinks.community.timeseries.query.Aggregation; +import org.jetlinks.community.timeseries.query.AggregationData; +import org.jetlinks.reactor.ql.utils.CastUtils; +import org.slf4j.Logger; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.util.*; +import java.util.function.Function; + +import static org.jetlinks.community.timescaledb.thing.TimescaleDBColumnModeQueryOperations.doAggregation0; + +@Slf4j +public class TimescaleDBRowModeQueryOperations extends RowModeQueryOperationsBase { + private final DatabaseOperator database; + + public TimescaleDBRowModeQueryOperations(String thingType, + String thingTemplateId, + String thingId, + MetricBuilder metricBuilder, + DataSettings settings, + ThingsRegistry registry, + DatabaseOperator database) { + super(thingType, thingTemplateId, thingId, metricBuilder, settings, registry); + this.database = database; + } + + @Override + protected Flux doQuery(String metric, Query query) { + + return query + .execute( + database + .dml() + .createReactiveRepository(TimescaleDBUtils.getTableName(metric)) + .createQuery()::setParam + ) + .fetch() + .map(this::convertToTimeSeriesData) + .contextWrite(ctx -> ctx.put(Logger.class, log)); + } + + @Override + protected Mono> doQueryPage(String metric, + Query query, + Function mapper) { + return QueryHelper + .queryPager( + query.getParam(), + () -> database + .dml() + .createReactiveRepository(TimescaleDBUtils.getTableName(metric)) + .createQuery(), + record -> mapper.apply(convertToTimeSeriesData(record)) + ) + .contextWrite(ctx -> ctx.put(Logger.class, log)); + } + + static final String timestampAlias = "_ts"; + + @Override + protected Flux doAggregation(String metric, + AggregationRequest request, + AggregationContext context) { + metric = TimescaleDBUtils.getTableName(metric); + QueryParamEntity filter = request.getFilter(); + filter.setSorts(new ArrayList<>()); + filter.setPaging(false); + QueryOperator query = database.dml().query(metric); + + SelectColumn propertyColumn = SelectColumn.of(ThingsDataConstants.COLUMN_PROPERTY_ID); + + query.groupBy(propertyColumn); + + //按时间分组 + if (request.getInterval() != null) { + NativeSelectColumn column = TimescaleDBUtils.createTimeGroupColumn( + request.getFrom().getTime(), + request.getInterval() + ); + + query.groupBy(column); + query.select(column); + column.setAlias(timestampAlias); + } + + query.select(propertyColumn); + + Set propertyId = new HashSet<>(); + + if (context.getProperties().length > 1) { + for (PropertyAggregation property : context.getProperties()) { + NativeSelectColumn column = new NativeSelectColumn( + "case when property = ? then " + createAggFunction(property.getAgg()) + + " end as \"" + property.getAlias() + "\""); + column.setParameters(new Object[]{property.getProperty()}); + query.select(column); + propertyId.add(property.getProperty()); + } + } else { + PropertyAggregation property = context.getProperties()[0]; + String sql = createAggFunction(property.getAgg()) + " as \"" + property.getAlias() + "\""; + query.select(NativeSelectColumn.of(sql)); + propertyId.add(property.getProperty()); + + } + + + query.where(cdt -> { + cdt.each(filter.getTerms(), Conditional::accept); + cdt.between(ThingsDataConstants.COLUMN_TIMESTAMP, request.getFrom(), request.getTo()); + if (!propertyId.isEmpty()) { + cdt.in(ThingsDataConstants.COLUMN_PROPERTY_ID, propertyId); + } + }); + + NavigableMap> + prepares = ThingsDataUtils.prepareAggregationData(request, context.getProperties()); + + return query + .fetch(ResultWrappers.map()) + .reactive() + .map(val -> Maps.filterValues(val, Objects::nonNull)) + .map(AggregationData::of) + .groupBy(data -> data.getLong(timestampAlias).orElse(0L), Integer.MAX_VALUE) + .flatMap(group -> { + long time = group.key(); + Map prepare = ThingsDataUtils.findAggregationData(time, prepares); + if (prepare == null) { + return Mono.empty(); + } + return group + .doOnNext(data -> { + for (PropertyAggregation property : context.getProperties()) { + String alias = property.getAlias(); + data.get(alias) + .ifPresent(val -> prepare.put(alias, val)); + } + }); + }) + .thenMany(Flux.fromIterable(prepares.values())) + .map(AggregationData::of) + .take((long) request.getLimit() * propertyId.size()) + .contextWrite(ctx -> ctx.put(Logger.class, log)); + } + + + private TimeSeriesData convertToTimeSeriesData(Record record) { + return TimeSeriesData.of( + record.get(ThingsDataConstants.COLUMN_TIMESTAMP) + .map(val -> CastUtils.castNumber(val).longValue()) + .orElseGet(System::currentTimeMillis), + record + ); + } + + + protected String createAggFunction(Aggregation aggregation) { + switch (aggregation) { + case COUNT: + return "count(1)"; + case DISTINCT_COUNT: + return "count(distinct \"numberValue\")"; + case AVG: + return "avg(\"numberValue\")"; + case MAX: + return "max(\"numberValue\")"; + case MIN: + return "min(\"numberValue\")"; + case SUM: + return "sum(\"numberValue\")"; +// case STDDEV: +// return "stddev(\"numberValue\")"; + } + throw new UnsupportedOperationException("不支持的聚合函数:" + aggregation); + } +} diff --git a/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/thing/TimescaleDBRowModeSaveOperations.java b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/thing/TimescaleDBRowModeSaveOperations.java new file mode 100644 index 00000000..fa6bccbc --- /dev/null +++ b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/thing/TimescaleDBRowModeSaveOperations.java @@ -0,0 +1,35 @@ +package org.jetlinks.community.timescaledb.thing; + +import org.jetlinks.core.things.ThingsRegistry; +import org.jetlinks.community.things.data.operations.DataSettings; +import org.jetlinks.community.things.data.operations.MetricBuilder; +import org.jetlinks.community.things.data.operations.RowModeSaveOperationsBase; +import org.jetlinks.community.timescaledb.TimescaleDBDataWriter; +import org.jetlinks.community.timescaledb.TimescaleDBUtils; +import org.jetlinks.community.timeseries.TimeSeriesData; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +public class TimescaleDBRowModeSaveOperations extends RowModeSaveOperationsBase { + + private final TimescaleDBDataWriter writer; + + public TimescaleDBRowModeSaveOperations(ThingsRegistry registry, + MetricBuilder metricBuilder, + DataSettings settings, + TimescaleDBDataWriter writer) { + super(registry, metricBuilder, settings); + this.writer = writer; + } + + @Override + protected Mono doSave(String metric, TimeSeriesData data) { + + return writer.save(TimescaleDBUtils.getTableName(metric), data.getData()); + } + + @Override + protected Mono doSave(String metric, Flux data) { + return writer.save(TimescaleDBUtils.getTableName(metric), data.map(TimeSeriesData::getData)); + } +} diff --git a/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/thing/TimescaleDBRowModeStrategy.java b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/thing/TimescaleDBRowModeStrategy.java new file mode 100644 index 00000000..30194f8d --- /dev/null +++ b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/thing/TimescaleDBRowModeStrategy.java @@ -0,0 +1,72 @@ +package org.jetlinks.community.timescaledb.thing; + +import org.jetlinks.core.things.ThingsRegistry; +import org.jetlinks.community.things.data.AbstractThingDataRepositoryStrategy; +import org.jetlinks.community.things.data.operations.DDLOperations; +import org.jetlinks.community.things.data.operations.QueryOperations; +import org.jetlinks.community.things.data.operations.SaveOperations; +import org.jetlinks.community.things.data.operations.TableSafeMetricBuilder; +import org.jetlinks.community.timescaledb.TimescaleDBOperations; + +public class TimescaleDBRowModeStrategy extends AbstractThingDataRepositoryStrategy { + + private final ThingsRegistry registry; + private final TimescaleDBOperations operations; + private final TimescaleDBThingsDataProperties properties; + + public TimescaleDBRowModeStrategy(ThingsRegistry registry, + TimescaleDBOperations operations, + TimescaleDBThingsDataProperties properties) { + this.registry = registry; + this.operations = operations; + this.properties = properties; + } + + @Override + public String getId() { + return "timescaledb-row"; + } + + @Override + public String getName() { + return "TimescaleDB-单列模式"; + } + + @Override + public SaveOperations createOpsForSave(OperationsContext context) { + return new TimescaleDBRowModeSaveOperations( + registry, + TableSafeMetricBuilder.of(context.getMetricBuilder()), + context.getSettings(), + operations.writer()); + } + + @Override + protected QueryOperations createForQuery(String thingType, String templateId, String thingId, OperationsContext context) { + return new TimescaleDBRowModeQueryOperations( + thingType, + templateId, + thingId, + TableSafeMetricBuilder.of(context.getMetricBuilder()), + context.getSettings(), + registry, + operations.database()); + } + + @Override + protected DDLOperations createForDDL(String thingType, String templateId, String thingId, OperationsContext context) { + return new TimescaleDBRowModeDDLOperations( + thingType, + templateId, + thingId, + context.getSettings(), + TableSafeMetricBuilder.of(context.getMetricBuilder()), + operations.database(), + properties); + } + + @Override + public int getOrder() { + return 10060; + } +} diff --git a/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/thing/TimescaleDBThingsDataProperties.java b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/thing/TimescaleDBThingsDataProperties.java new file mode 100644 index 00000000..c5c4ecde --- /dev/null +++ b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/thing/TimescaleDBThingsDataProperties.java @@ -0,0 +1,23 @@ +package org.jetlinks.community.timescaledb.thing; + +import lombok.Getter; +import lombok.Setter; +import org.jetlinks.community.Interval; +import org.springframework.boot.context.properties.ConfigurationProperties; + +@Getter +@Setter +@ConfigurationProperties(prefix = "timescaledb.things-data") +public class TimescaleDBThingsDataProperties { + private boolean enabled = true; + + /** + * 分区时间间隔 + */ + private Interval chunkTimeInterval = Interval.ofDays(7); + + /** + * 数据保留时长 + */ + private Interval retentionPolicy; +} diff --git a/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/timeseries/TimescaleDBTimeSeriesManager.java b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/timeseries/TimescaleDBTimeSeriesManager.java new file mode 100644 index 00000000..acd12ed3 --- /dev/null +++ b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/timeseries/TimescaleDBTimeSeriesManager.java @@ -0,0 +1,127 @@ +package org.jetlinks.community.timescaledb.timeseries; + +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.hswebframework.ezorm.rdb.codec.DateTimeCodec; +import org.hswebframework.ezorm.rdb.metadata.RDBIndexMetadata; +import org.hswebframework.ezorm.rdb.operator.ddl.TableBuilder; +import org.jetlinks.core.metadata.PropertyMetadata; +import org.jetlinks.community.Interval; +import org.jetlinks.community.things.data.ThingsDataConstants; +import org.jetlinks.community.things.utils.ThingsDatabaseUtils; +import org.jetlinks.community.timescaledb.TimescaleDBOperations; +import org.jetlinks.community.timescaledb.TimescaleDBUtils; +import org.jetlinks.community.timescaledb.metadata.CreateHypertable; +import org.jetlinks.community.timescaledb.metadata.CreateRetentionPolicy; +import org.jetlinks.community.timeseries.TimeSeriesManager; +import org.jetlinks.community.timeseries.TimeSeriesMetadata; +import org.jetlinks.community.timeseries.TimeSeriesMetric; +import org.jetlinks.community.timeseries.TimeSeriesService; +import org.slf4j.Logger; +import reactor.core.publisher.Mono; + +import java.sql.JDBCType; +import java.util.Date; + +import static org.jetlinks.community.ConfigMetadataConstants.indexEnabled; + +@Slf4j +@AllArgsConstructor +public class TimescaleDBTimeSeriesManager implements TimeSeriesManager { + + private final TimescaleDBTimeSeriesProperties properties; + + private final TimescaleDBOperations operations; + + @Override + public TimeSeriesService getService(TimeSeriesMetric metric) { + return getService(metric.getId()); + } + + @Override + public TimeSeriesService getServices(TimeSeriesMetric... metric) { + if (metric.length == 1) { + return getServices(metric[0].getId()); + } + throw new UnsupportedOperationException("unsupported multi metric"); + } + + @Override + public TimeSeriesService getServices(String... metric) { + if (metric.length == 1) { + return getService(metric[0]); + } + throw new UnsupportedOperationException("unsupported multi metric"); + } + + @Override + public TimeSeriesService getService(String metric) { + return new TimescaleDBTimeSeriesService(TimescaleDBUtils.getTableName(metric), operations); + } + + @Override + public Mono registerMetadata(TimeSeriesMetadata metadata) { + String tableName = TimescaleDBUtils.getTableName(metadata.getMetric().getId()); + TableBuilder builder = operations + .database() + .ddl() + .createOrAlter(tableName) + .custom(table -> { + table.addFeature(new CreateHypertable(ThingsDataConstants.COLUMN_TIMESTAMP, properties.getChunkTimeInterval())); + Interval interval = properties.getRetentionPolicy(tableName); + if (interval != null && interval.getNumber().longValue() > 0) { + table.addFeature(new CreateRetentionPolicy(interval)); + } + }); + for (PropertyMetadata property : metadata.getProperties()) { + builder + .addColumn(property.getId()) + .custom(column -> { + ThingsDatabaseUtils.convertColumn(property, column); + TimescaleDBUtils.customColumn(property, column); + }) + .commit(); + //开启索引 + if (property.getExpand(indexEnabled).orElse(false)) { + builder + .index() + .name("idx_" + tableName + "_" + property.getId()) + .column(property.getId()) + .commit(); + } + } + builder + .addColumn(ThingsDataConstants.COLUMN_ID) + .comment("ID") + .varchar(64) + .notNull() + .commit(); + + builder + .addColumn(ThingsDataConstants.COLUMN_TIMESTAMP) + .comment("时间戳") + .custom(column -> { + //column + column.setJdbcType(JDBCType.TIMESTAMP, Date.class); + column.setValueCodec(new DateTimeCodec("yyyy-MM-dd HH:mm:ss.SSS", Date.class)); + }) + .notNull() + .commit(); + + builder + .index() + .name("idx_" + tableName + "_id") + .column(ThingsDataConstants.COLUMN_ID) + .column(ThingsDataConstants.COLUMN_TIMESTAMP, RDBIndexMetadata.IndexSort.desc) + .unique() + .commit(); + + return builder + .commit() + .reactive() + .then() + .contextWrite(ctx -> ctx.put(Logger.class, log)); + } + + +} diff --git a/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/timeseries/TimescaleDBTimeSeriesProperties.java b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/timeseries/TimescaleDBTimeSeriesProperties.java new file mode 100644 index 00000000..dd7e5bf6 --- /dev/null +++ b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/timeseries/TimescaleDBTimeSeriesProperties.java @@ -0,0 +1,51 @@ +package org.jetlinks.community.timescaledb.timeseries; + +import lombok.Getter; +import lombok.Setter; +import org.jetlinks.community.Interval; +import org.jetlinks.community.timescaledb.TimescaleDBUtils; +import org.springframework.boot.context.properties.ConfigurationProperties; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +@Getter +@Setter +@ConfigurationProperties(prefix = "timescaledb.time-series") +public class TimescaleDBTimeSeriesProperties { + private boolean enabled = true; + + /** + * 分区时间间隔 + */ + private Interval chunkTimeInterval = Interval.ofDays(7); + + /** + * 对特定的表设置数据保留时长 + */ + private List retentionPolicies = new ArrayList<>(); + + /** + * 默认数据保留时长 + */ + private Interval retentionPolicy; + + + public Interval getRetentionPolicy(String tableName) { + + for (RetentionPolicy policy : retentionPolicies) { + if (Objects.equals(TimescaleDBUtils.getTableName(policy.table), tableName)) { + return policy.interval; + } + } + return retentionPolicy; + } + + @Getter + @Setter + public static class RetentionPolicy { + private String table; + private Interval interval; + } +} diff --git a/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/timeseries/TimescaleDBTimeSeriesService.java b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/timeseries/TimescaleDBTimeSeriesService.java new file mode 100644 index 00000000..8b41edb6 --- /dev/null +++ b/jetlinks-components/timescaledb-component/src/main/java/org/jetlinks/community/timescaledb/timeseries/TimescaleDBTimeSeriesService.java @@ -0,0 +1,191 @@ +package org.jetlinks.community.timescaledb.timeseries; + +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.hswebframework.ezorm.core.param.QueryParam; +import org.hswebframework.ezorm.rdb.executor.wrapper.ResultWrappers; +import org.hswebframework.ezorm.rdb.operator.dml.QueryOperator; +import org.hswebframework.ezorm.rdb.operator.dml.query.NativeSelectColumn; +import org.hswebframework.ezorm.rdb.operator.dml.query.SelectColumn; +import org.hswebframework.web.bean.FastBeanCopier; +import org.hswebframework.web.id.IDGenerator; +import org.jetlinks.core.utils.Reactors; +import org.jetlinks.community.things.data.ThingsDataConstants; +import org.jetlinks.community.timescaledb.TimescaleDBOperations; +import org.jetlinks.community.timescaledb.TimescaleDBUtils; +import org.jetlinks.community.timeseries.TimeSeriesData; +import org.jetlinks.community.timeseries.TimeSeriesService; +import org.jetlinks.community.timeseries.query.*; +import org.jetlinks.community.timeseries.utils.TimeSeriesUtils; +import org.jetlinks.reactor.ql.utils.CastUtils; +import org.reactivestreams.Publisher; +import org.slf4j.Logger; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; + +@Slf4j +@AllArgsConstructor +public class TimescaleDBTimeSeriesService implements TimeSeriesService { + private final String metric; + private final TimescaleDBOperations operations; + + @Override + public Flux query(QueryParam queryParam) { + if (isTableNotExist()) { + return Flux.empty(); + } + return operations + .database() + .dml() + .createReactiveRepository(metric) + .createQuery() + .setParam(queryParam) + .fetch() + .map(TimescaleDBUtils::convertToTimeSeriesData) + .contextWrite(ctx -> ctx.put(Logger.class, log)); + } + + @Override + public Flux multiQuery(Collection query) { + return Flux.fromIterable(query) + .flatMap(this::query, 8); + } + + @Override + public Mono count(QueryParam queryParam) { + if (isTableNotExist()) { + return Reactors.ALWAYS_ZERO; + } + return operations + .database() + .dml() + .createReactiveRepository(metric) + .createQuery() + .setParam(queryParam) + .count() + .contextWrite(ctx -> ctx.put(Logger.class, log)); + } + + @Override + public Flux aggregation(AggregationQueryParam param) { + List groups = param.getGroups(); + TimeSeriesUtils.prepareAggregationQueryParam(param); + + long startWith = param.getStartWithTime(); + if (isTableNotExist()) { + return Flux.empty(); + } + + QueryOperator query = operations + .database() + .dml() + .query(metric) + .setParam(param.getQueryParam().noPaging()) + .where(conditional -> conditional.between(ThingsDataConstants.COLUMN_TIMESTAMP, startWith, param.getEndWithTime())); + + for (AggregationColumn aggColumn : param.getAggColumns()) { + SelectColumn column = SelectColumn + .of(aggColumn.getProperty(), + aggColumn.getAlias()); + TimescaleDBUtils.applyAggColumn(aggColumn.getAggregation(), column); + + query.select(column); + } + TimeGroup _timeGroup = null; + for (Group group : groups) { + if (group instanceof TimeGroup) { + _timeGroup = ((TimeGroup) group); + NativeSelectColumn column = TimescaleDBUtils.createTimeGroupColumn(startWith, _timeGroup.getInterval()); + column.setColumn(ThingsDataConstants.COLUMN_TIMESTAMP); + column.setAlias(group.getAlias()); + query.select(column); + query.groupBy(column); + } else { + SelectColumn column = SelectColumn.of(group.getProperty(), group.getAlias()); + query.select(column); + query.groupBy(column); + } + } + TimeGroup timeGroup = _timeGroup; + Map, NavigableMap>> + groupedResults = new ConcurrentHashMap<>(); + Set key = new LinkedHashSet<>(); + //无数据时返回默认统计，供前端显示坐标轴 + groupedResults + .computeIfAbsent(key, (ignore) -> TimeSeriesUtils.prepareAggregationData(param, timeGroup)); + + return query + .fetch(ResultWrappers.map()) + .reactive() + .as(flux -> { + if (groups.isEmpty() || timeGroup == null) { + return flux; + } + + return flux + .doOnNext(data -> { + for (Group group : groups) { + if (group instanceof TimeGroup) { + continue; + } + Object g = data.get(group.getAlias()); + key.add(g); + } + NavigableMap> prepares = groupedResults + .computeIfAbsent(key, (ignore) -> TimeSeriesUtils.prepareAggregationData(param, timeGroup)); + long ts = CastUtils.castNumber(data.getOrDefault(timeGroup.getAlias(), 0)).longValue(); + Map prepare = TimeSeriesUtils.findAggregationData(ts, prepares); + //时间错误? + if (prepare == null) { + return; + } + FastBeanCopier.copy(data, prepare, timeGroup.getAlias()); + }) + .thenMany(Flux.defer(() -> Flux.fromIterable(groupedResults.values()))) + .flatMapIterable(Map::values); + }) + .map(AggregationData::of) + .contextWrite(ctx -> ctx.put(Logger.class, log)); + } + + private boolean isTableNotExist() { + return !operations.database() + .getMetadata() + .getTableOrView(metric, false) + .isPresent(); + } + + @Override + public Mono commit(Publisher data) { + return Flux + .from(data) + .flatMap(this::commit) + .then(); + } + + @Override + public Mono commit(TimeSeriesData data) { + + return operations.writer().save(metric, toTimeSeriesData(data)); + } + + @Override + public Mono save(Publisher data) { + + return operations.writer().save(metric, Flux + .from(data) + .map(TimescaleDBTimeSeriesService::toTimeSeriesData)) + ; + } + + private static Map toTimeSeriesData(TimeSeriesData data) { + Map map = data.getData(); + map.put(ThingsDataConstants.COLUMN_TIMESTAMP, data.getTimestamp()); + map.computeIfAbsent(ThingsDataConstants.COLUMN_ID, ignore -> IDGenerator.RANDOM.generate()); + + return data.getData(); + } +} diff --git a/jetlinks-components/timescaledb-component/src/main/resources/META-INF/services/org.hswebframework.web.crud.configuration.DialectProvider b/jetlinks-components/timescaledb-component/src/main/resources/META-INF/services/org.hswebframework.web.crud.configuration.DialectProvider new file mode 100644 index 00000000..0d0afac9 --- /dev/null +++ b/jetlinks-components/timescaledb-component/src/main/resources/META-INF/services/org.hswebframework.web.crud.configuration.DialectProvider @@ -0,0 +1 @@ +org.jetlinks.community.timescaledb.metadata.TimescaleDBDialectProvider \ No newline at end of file diff --git a/jetlinks-components/timescaledb-component/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/jetlinks-components/timescaledb-component/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports new file mode 100644 index 00000000..8cadb460 --- /dev/null +++ b/jetlinks-components/timescaledb-component/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports @@ -0,0 +1,3 @@ +org.jetlinks.community.timescaledb.configuration.TimescaleDBConfiguration +org.jetlinks.community.timescaledb.configuration.TimescaleDBThingsDataConfiguration +org.jetlinks.community.timescaledb.configuration.TimescaleDBTimeSeriesConfiguration \ No newline at end of file diff --git a/jetlinks-components/timeseries-component/pom.xml b/jetlinks-components/timeseries-component/pom.xml index d993de7d..54d37fe5 100644 --- a/jetlinks-components/timeseries-component/pom.xml +++ b/jetlinks-components/timeseries-component/pom.xml @@ -5,7 +5,7 @@ jetlinks-components org.jetlinks.community - 2.3.0-SNAPSHOT + 2.10.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/jetlinks-components/timeseries-component/src/main/java/org/jetlinks/community/timeseries/query/Aggregation.java b/jetlinks-components/timeseries-component/src/main/java/org/jetlinks/community/timeseries/query/Aggregation.java index 1c0ff079..d4a291ef 100644 --- a/jetlinks-components/timeseries-component/src/main/java/org/jetlinks/community/timeseries/query/Aggregation.java +++ b/jetlinks-components/timeseries-component/src/main/java/org/jetlinks/community/timeseries/query/Aggregation.java @@ -4,6 +4,7 @@ import lombok.AllArgsConstructor; import lombok.Getter; import org.hswebframework.web.dict.EnumDict; import org.jetlinks.core.utils.Reactors; +import org.jetlinks.reactor.ql.utils.CastUtils; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.math.MathFlux; @@ -15,13 +16,14 @@ import java.util.function.Function; @AllArgsConstructor public enum Aggregation implements EnumDict { - MIN(numberFlux -> MathFlux.min(numberFlux, Comparator.comparing(Number::doubleValue)), null), - MAX(numberFlux -> MathFlux.max(numberFlux, Comparator.comparing(Number::doubleValue)),null), - AVG(numberFlux -> MathFlux.averageDouble(numberFlux, Number::doubleValue),null), - SUM(numberFlux -> MathFlux.sumDouble(numberFlux, Number::doubleValue),0), - COUNT(Flux::count, 0), - FIRST(numberFlux -> numberFlux.take(1).single(),null), - TOP(numberFlux -> numberFlux.take(1).single(),null), + MIN(numberFlux -> MathFlux.min(numberFlux.map(CastUtils::castNumber), Comparator.comparing(Number::doubleValue)), null), + MAX(numberFlux -> MathFlux.max(numberFlux.map(CastUtils::castNumber), Comparator.comparing(Number::doubleValue)),null), + AVG(numberFlux -> MathFlux.averageDouble(numberFlux.map(CastUtils::castNumber), Number::doubleValue),null), + SUM(numberFlux -> MathFlux.sumDouble(numberFlux.map(CastUtils::castNumber), Number::doubleValue),0), + COUNT(Flux::count,0), + FIRST(numberFlux -> numberFlux.take(1).singleOrEmpty(),null), + TOP(numberFlux -> numberFlux.take(1).singleOrEmpty(),null), + LAST(numberFlux -> numberFlux.takeLast(1).singleOrEmpty(),null), //去重计数 DISTINCT_COUNT(flux -> flux.distinct().count(),0), @@ -51,4 +53,11 @@ public enum Aggregation implements EnumDict { return false; } + public boolean needNumberValue() { + boolean defaultValue = COUNT == this + || FIRST == this + || LAST == this + || TOP == this; + return !defaultValue; + } } \ No newline at end of file diff --git a/jetlinks-components/timeseries-component/src/main/java/org/jetlinks/community/timeseries/query/AggregationColumn.java b/jetlinks-components/timeseries-component/src/main/java/org/jetlinks/community/timeseries/query/AggregationColumn.java index 6edcf632..ffeafc80 100644 --- a/jetlinks-components/timeseries-component/src/main/java/org/jetlinks/community/timeseries/query/AggregationColumn.java +++ b/jetlinks-components/timeseries-component/src/main/java/org/jetlinks/community/timeseries/query/AggregationColumn.java @@ -7,8 +7,8 @@ import lombok.NoArgsConstructor; import lombok.Setter; import org.springframework.util.StringUtils; -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.NotNull; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; @Getter @Setter diff --git a/jetlinks-components/timeseries-component/src/main/java/org/jetlinks/community/timeseries/utils/TimeSeriesUtils.java b/jetlinks-components/timeseries-component/src/main/java/org/jetlinks/community/timeseries/utils/TimeSeriesUtils.java new file mode 100644 index 00000000..c3e97c1b --- /dev/null +++ b/jetlinks-components/timeseries-component/src/main/java/org/jetlinks/community/timeseries/utils/TimeSeriesUtils.java @@ -0,0 +1,109 @@ +package org.jetlinks.community.timeseries.utils; + +import org.hswebframework.ezorm.core.param.Term; +import org.hswebframework.ezorm.core.param.TermType; +import org.jetlinks.core.metadata.types.DateTimeType; +import org.jetlinks.community.Interval; +import org.jetlinks.community.timeseries.query.AggregationColumn; +import org.jetlinks.community.timeseries.query.AggregationQueryParam; +import org.jetlinks.community.timeseries.query.TimeGroup; +import org.jetlinks.reactor.ql.utils.CastUtils; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; +import org.joda.time.Instant; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; + +import java.time.Duration; +import java.util.*; +import java.util.function.BiFunction; + +public class TimeSeriesUtils { + + + public static Map findAggregationData(long ts, + NavigableMap> container) { + + Map prepare = container.get(ts); + if (prepare != null) { + return prepare; + } + Map.Entry> entry = container.floorEntry(ts); + if (entry != null) { + return entry.getValue(); + } + return null; + } + + public static long truncateTime(long time, Interval interval) { + return interval.getUnit().truncatedTo(time); + } + + public static NavigableMap> prepareAggregationData(AggregationQueryParam request, + TimeGroup group) { + + return prepareAggregationData(request, group, TimeSeriesUtils::truncateTime); + } + + public static NavigableMap> prepareAggregationData(AggregationQueryParam request, + TimeGroup group, + BiFunction timeTruncate) { + NavigableMap> data = new TreeMap<>(Comparator.comparingLong(l -> -l)); + Map valueMap = new HashMap<>(); + for (AggregationColumn property : request.getAggColumns()) { + valueMap.put(property.getAlias(), property.getAggregation().getDefaultValue()); + } + if (group == null || group.getInterval() == null) { + data.put(0L, valueMap); + return data; + } + DateTimeFormatter formatter = DateTimeFormat.forPattern(group.getFormat()); + + long startWith = request.getStartWithTime(); + long endWith = request.getEndWithTime(); + + Iterable iterable = group.getInterval().iterate(startWith, endWith); + for (Long time : iterable) { + time = timeTruncate.apply(time, group.getInterval()); + Map tempData = new HashMap<>(valueMap); + String formatValue = new DateTime(new Instant(time), DateTimeZone.getDefault()).toString(formatter); + tempData.put(group.getAlias(), formatValue); + data.put(time, tempData); + } + + return data; + } + + public static void prepareAggregationQueryParam(AggregationQueryParam param) { + if (param.getStartWithTime() == 0) { + param.setStartWithTime(calculateStartWithTime(param)); + } + } + + public static long calculateStartWithTime(AggregationQueryParam param) { + long startWithParam = param.getStartWithTime(); + if (startWithParam == 0) { + //从查询条件中提取时间参数来获取时间区间 + List terms = param.getQueryParam().getTerms(); + for (Term term : terms) { + if ("timestamp".equals(term.getColumn())) { + Object value = term.getValue(); + String termType = term.getTermType(); + if (TermType.btw.equals(termType)) { + if (String.valueOf(value).contains(",")) { + value = Arrays.asList(String.valueOf(value).split(",")); + } + return DateTimeType.GLOBAL.convert(CastUtils.castArray(value).get(0)).getTime(); + } + if (TermType.gt.equals(termType) || TermType.gte.equals(termType)) { + + return DateTimeType.GLOBAL.convert(value).getTime(); + } + } + } + return param.getEndWithTime() - Duration.ofDays(90).toMillis(); + } + return startWithParam; + } + +} diff --git a/jetlinks-manager/authentication-manager/pom.xml b/jetlinks-manager/authentication-manager/pom.xml index b920c670..5c949569 100644 --- a/jetlinks-manager/authentication-manager/pom.xml +++ b/jetlinks-manager/authentication-manager/pom.xml @@ -7,7 +7,7 @@ org.jetlinks.community jetlinks-manager - 2.3.0-SNAPSHOT + 2.10.0-SNAPSHOT ../pom.xml authentication-manager diff --git a/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/configuration/AuthorizationPermissionInitializeService.java b/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/configuration/AuthorizationPermissionInitializeService.java new file mode 100644 index 00000000..d95d65b5 --- /dev/null +++ b/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/configuration/AuthorizationPermissionInitializeService.java @@ -0,0 +1,106 @@ +package org.jetlinks.community.auth.configuration; + +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.hswebframework.web.authorization.Authentication; +import org.hswebframework.web.authorization.Dimension; +import org.hswebframework.web.authorization.Permission; +import org.hswebframework.web.authorization.User; +import org.hswebframework.web.authorization.events.AuthorizationInitializeEvent; +import org.hswebframework.web.authorization.simple.SimpleAuthentication; +import org.hswebframework.web.authorization.simple.SimplePermission; +import org.hswebframework.web.authorization.simple.SimpleUser; +import org.jetlinks.community.utils.ConverterUtils; +import org.springframework.context.event.EventListener; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; + +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * 根据配置进行权限初始化 + */ +@Slf4j +@AllArgsConstructor +public class AuthorizationPermissionInitializeService { + + private final AuthorizationProperties authorizationProperties; + + @EventListener + public void handleAuthInitEvent(AuthorizationInitializeEvent event) { + Authentication before = event.getAuthentication(); + List defaultPermission = before + .getDimensions() + .stream() + .flatMap(this::getPermission) + .collect(Collectors.toList()); + + if (CollectionUtils.isNotEmpty(defaultPermission)) { + SimpleAuthentication defaultAuth = new SimpleAuthentication(); + defaultAuth.setPermissions(defaultPermission); + defaultAuth.merge(before); + event.setAuthentication(defaultAuth); + } + } + + + public Stream getPermission(Dimension dimension) { + MultiValueMap map = new LinkedMultiValueMap<>(); + Optional + .ofNullable(authorizationProperties.getDefaults().get(dimension.getType().getId())) + .map(dimensionPermission -> { + String dimensionKey = dimension.getId(); + if (dimension instanceof User) { + //用户类型根据username获取 + dimensionKey = ((SimpleUser) dimension).getUsername(); + } + return dimensionPermission.get(dimensionKey); + }) + .orElse(Collections.emptyList()) + .forEach(text -> parse(text, map)); + + return map + .entrySet() + .stream() + .map(entry -> { + String id = entry.getKey(); + Set actions = entry + .getValue() + .stream() + .filter(StringUtils::isNotEmpty) + .collect(Collectors.toSet()); + return SimplePermission + .builder() + .id(id) + .name(id) + .actions(actions) + .build(); + }); + } + + /** + * 解析permissionText + * + * @param text: permissionId:action1,action2 例如device-instance:query,save + */ + public static void parse(String text, MultiValueMap map) { + String[] split = text.split(":", 2); + if (split.length == 2) { + String permissionId = split[0]; + map.add(permissionId, null); + + if (StringUtils.isNotEmpty(split[1])) { + ConverterUtils + .convertToList(split[1].split(","), String::valueOf) + .forEach(action -> map.add(permissionId, action)); + } + } + } +} \ No newline at end of file diff --git a/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/configuration/AuthorizationProperties.java b/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/configuration/AuthorizationProperties.java new file mode 100644 index 00000000..cac9d6a4 --- /dev/null +++ b/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/configuration/AuthorizationProperties.java @@ -0,0 +1,46 @@ +package org.jetlinks.community.auth.configuration; + +import lombok.Getter; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.context.properties.ConfigurationProperties; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * 认证信息相关配置项 + * 
{@code
+ * authentication:
+ *   defaults:
+ *     user:
+ *       userName-demo:
+ *         - "*:*"
+ *     role:
+ *       roleId-demo:
+ *         - "device-instance:query,save"
+ *     org:
+ *       orgId-demo:
+ *         - "device-instance:query,save"
+ * }
+ *
+ * + * @author gyl + * @since 2.2 + */ +@Slf4j +@ConfigurationProperties(prefix = "jetlinks.authentication") +@Getter +@Setter +public class AuthorizationProperties { + + /** + * 指定维度类型及维度id配置默认权限 + *

在认证信息初始化时，填充合并所在维度的默认权限 + * + * @see AuthorizationPermissionInitializeService + */ + private Map>> defaults = new HashMap<>(); + +} \ No newline at end of file diff --git a/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/configuration/CustomAuthenticationConfiguration.java b/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/configuration/CustomAuthenticationConfiguration.java index 45e5fb82..df3852eb 100644 --- a/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/configuration/CustomAuthenticationConfiguration.java +++ b/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/configuration/CustomAuthenticationConfiguration.java @@ -20,7 +20,10 @@ import org.springframework.data.redis.core.ReactiveRedisOperations; import java.time.Duration; @Configuration(proxyBeanMethods = false) -@EnableConfigurationProperties({MenuProperties.class}) +@EnableConfigurationProperties({ + MenuProperties.class, + AuthorizationProperties.class +}) public class CustomAuthenticationConfiguration { static final String CONDITION_CLASS_NAME = "org.jetlinks.community.microservice.configuration.CloudServicesConfiguration"; @@ -51,6 +54,11 @@ public class CustomAuthenticationConfiguration { return new UserAuthenticationEventPublisher(eventBus); } + @Bean + public AuthorizationPermissionInitializeService authorizationPermissionInitializeService(AuthorizationProperties properties){ + return new AuthorizationPermissionInitializeService(properties); + } + @Bean public Jackson2ObjectMapperBuilderCustomizer jackson2ObjectMapperBuilderAuthCustomizer() { return builder -> { diff --git a/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/entity/MenuBindEntity.java b/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/entity/MenuBindEntity.java index 5cd5964a..3c97f915 100644 --- a/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/entity/MenuBindEntity.java +++ b/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/entity/MenuBindEntity.java @@ -18,7 +18,7 @@ import org.springframework.util.ObjectUtils; import javax.persistence.Column; import javax.persistence.Index; import javax.persistence.Table; -import javax.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotBlank; import java.sql.JDBCType; import java.util.List; import java.util.Map; diff --git a/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/entity/OrganizationEntity.java b/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/entity/OrganizationEntity.java index 50515016..c4d08b4b 100644 --- a/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/entity/OrganizationEntity.java +++ b/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/entity/OrganizationEntity.java @@ -21,7 +21,7 @@ import org.jetlinks.core.things.ThingInfo; import javax.persistence.Column; import javax.persistence.Table; -import javax.validation.constraints.Pattern; +import jakarta.validation.constraints.Pattern; import java.sql.JDBCType; import java.util.Collections; import java.util.HashMap; diff --git a/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/entity/ThirdPartyUserBindEntity.java b/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/entity/ThirdPartyUserBindEntity.java index b6acb977..cc650fcc 100755 --- a/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/entity/ThirdPartyUserBindEntity.java +++ b/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/entity/ThirdPartyUserBindEntity.java @@ -16,7 +16,7 @@ import org.hswebframework.web.validator.CreateGroup; import javax.persistence.Column; import javax.persistence.Index; import javax.persistence.Table; -import javax.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotBlank; import java.sql.JDBCType; import java.util.Map; diff --git a/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/entity/UserDetailEntity.java b/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/entity/UserDetailEntity.java index 92b9cebc..7ce79749 100644 --- a/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/entity/UserDetailEntity.java +++ b/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/entity/UserDetailEntity.java @@ -19,9 +19,9 @@ import org.jetlinks.community.relation.RelationConstants; import javax.persistence.Column; import javax.persistence.Table; -import javax.validation.constraints.Email; -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.Pattern; +import jakarta.validation.constraints.Email; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.Pattern; /** * 用户信息 diff --git a/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/initialize/UserAutoInitialize.java b/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/initialize/UserAutoInitialize.java new file mode 100644 index 00000000..cf0a61a6 --- /dev/null +++ b/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/initialize/UserAutoInitialize.java @@ -0,0 +1,69 @@ +package org.jetlinks.community.auth.initialize; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.hswebframework.web.api.crud.entity.QueryParamEntity; +import org.hswebframework.web.system.authorization.api.entity.UserEntity; +import org.hswebframework.web.system.authorization.api.service.reactive.ReactiveUserService; +import org.springframework.boot.CommandLineRunner; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.util.Map; +import java.util.stream.Collectors; + +@Component +@ConditionalOnProperty(prefix = "jetlinks.user-init", + name = "enabled", + havingValue = "true") +@RequiredArgsConstructor +@Slf4j +public class UserAutoInitialize implements CommandLineRunner { + + private final UserAutoInitializeProperties properties; + + private final ReactiveUserService userService; + + @Override + public void run(String... args) throws Exception { + if (properties.isEnabled() && CollectionUtils.isNotEmpty(properties.getUsers())) { + Map mapping = properties.getUsers() + .stream() + .filter(user -> StringUtils.hasText(user.getUsername())) + .collect(Collectors.toMap(UserEntity::getUsername, user -> user)); + QueryParamEntity + .newQuery() + .in(UserEntity::getUsername, mapping.keySet()) + .execute(userService::findUser) + .doOnNext(u -> mapping.remove(u.getUsername())) + .then( + Mono.defer(() -> Flux + .fromIterable(mapping.values()) + .flatMap(user -> initAdminUser(user) + .onErrorResume(err -> { + log.error("init user [{}] error", user.getUsername(), err); + return Mono.empty(); + })) + .then()) + ) + .subscribe(); + + } + } + + private Mono initAdminUser(UserEntity entity) { + + entity.setCreatorId("system"); + if (entity.getName() == null) { + entity.setName(entity.getUsername()); + } + + return userService + .saveUser(Mono.just(entity)) + .thenReturn(entity); + } +} diff --git a/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/initialize/UserAutoInitializeProperties.java b/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/initialize/UserAutoInitializeProperties.java new file mode 100644 index 00000000..29bdd21b --- /dev/null +++ b/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/initialize/UserAutoInitializeProperties.java @@ -0,0 +1,20 @@ +package org.jetlinks.community.auth.initialize; + +import lombok.Getter; +import lombok.Setter; +import org.hswebframework.web.system.authorization.api.entity.UserEntity; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +import java.util.List; + +@Getter +@Setter +@ConfigurationProperties(prefix = "jetlinks.user-init") +@Component +public class UserAutoInitializeProperties { + + private boolean enabled; + + private List users; +} diff --git a/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/service/RoleService.java b/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/service/RoleService.java index 38649fb7..c71a5d9f 100644 --- a/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/service/RoleService.java +++ b/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/service/RoleService.java @@ -13,7 +13,7 @@ import org.springframework.transaction.annotation.Transactional; import reactor.core.publisher.Mono; import javax.annotation.Nullable; -import javax.validation.constraints.NotNull; +import jakarta.validation.constraints.NotNull; import java.util.Collection; @Service diff --git a/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/service/request/SaveUserDetailRequest.java b/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/service/request/SaveUserDetailRequest.java index 2966756e..c01f8f74 100644 --- a/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/service/request/SaveUserDetailRequest.java +++ b/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/service/request/SaveUserDetailRequest.java @@ -4,7 +4,7 @@ import io.swagger.v3.oas.annotations.media.Schema; import lombok.Getter; import lombok.Setter; -import javax.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotBlank; @Getter @Setter diff --git a/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/service/request/SaveUserRequest.java b/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/service/request/SaveUserRequest.java index 383ba24c..69f4f657 100644 --- a/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/service/request/SaveUserRequest.java +++ b/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/service/request/SaveUserRequest.java @@ -9,7 +9,7 @@ import org.hswebframework.web.validator.ValidatorUtils; import org.jetlinks.community.auth.entity.UserDetail; import org.springframework.util.StringUtils; -import javax.validation.constraints.NotNull; +import jakarta.validation.constraints.NotNull; import java.util.Set; /** diff --git a/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/web/request/AuthorizationSettingDetail.java b/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/web/request/AuthorizationSettingDetail.java index 42b1df20..04a23efa 100755 --- a/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/web/request/AuthorizationSettingDetail.java +++ b/jetlinks-manager/authentication-manager/src/main/java/org/jetlinks/community/auth/web/request/AuthorizationSettingDetail.java @@ -12,7 +12,7 @@ import org.hswebframework.web.system.authorization.api.entity.DataAccessEntity; import org.hswebframework.web.system.authorization.api.entity.PermissionEntity; import org.springframework.util.StringUtils; -import javax.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotBlank; import java.util.*; import java.util.stream.Collectors; diff --git a/jetlinks-manager/device-manager/pom.xml b/jetlinks-manager/device-manager/pom.xml index 4480c1fe..b30bbadc 100644 --- a/jetlinks-manager/device-manager/pom.xml +++ b/jetlinks-manager/device-manager/pom.xml @@ -7,7 +7,7 @@ org.jetlinks.community jetlinks-manager - 2.3.0-SNAPSHOT + 2.10.0-SNAPSHOT ../pom.xml device-manager @@ -38,7 +38,7 @@ ${project.groupId} - elasticsearch-component + timeseries-component ${project.version} diff --git a/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/entity/DeviceCategoryEntity.java b/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/entity/DeviceCategoryEntity.java index a22a073b..29a37507 100755 --- a/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/entity/DeviceCategoryEntity.java +++ b/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/entity/DeviceCategoryEntity.java @@ -18,8 +18,8 @@ import javax.persistence.Column; import javax.persistence.GeneratedValue; import javax.persistence.Id; import javax.persistence.Table; -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.Pattern; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.Pattern; import java.sql.JDBCType; import java.util.List; import java.util.Map; diff --git a/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/entity/DeviceInstanceEntity.java b/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/entity/DeviceInstanceEntity.java index cc2fd3a0..ef799ad7 100755 --- a/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/entity/DeviceInstanceEntity.java +++ b/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/entity/DeviceInstanceEntity.java @@ -36,8 +36,8 @@ import javax.persistence.Column; import javax.persistence.GeneratedValue; import javax.persistence.Index; import javax.persistence.Table; -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.Pattern; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.Pattern; import java.sql.JDBCType; import java.util.*; import java.util.stream.Stream; diff --git a/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/entity/DeviceMetadataMappingEntity.java b/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/entity/DeviceMetadataMappingEntity.java index ecc5d145..53c3282d 100644 --- a/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/entity/DeviceMetadataMappingEntity.java +++ b/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/entity/DeviceMetadataMappingEntity.java @@ -19,8 +19,8 @@ import org.springframework.util.StringUtils; import javax.persistence.Column; import javax.persistence.Index; import javax.persistence.Table; -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.NotNull; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; import java.sql.JDBCType; import java.util.Map; diff --git a/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/entity/DeviceProductEntity.java b/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/entity/DeviceProductEntity.java index fade55f7..fd43944d 100755 --- a/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/entity/DeviceProductEntity.java +++ b/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/entity/DeviceProductEntity.java @@ -34,8 +34,8 @@ import javax.persistence.Column; import javax.persistence.GeneratedValue; import javax.persistence.Index; import javax.persistence.Table; -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.Pattern; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.Pattern; import java.sql.JDBCType; import java.util.*; diff --git a/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/entity/DeviceTagEntity.java b/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/entity/DeviceTagEntity.java index 1be4c687..22c82dad 100644 --- a/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/entity/DeviceTagEntity.java +++ b/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/entity/DeviceTagEntity.java @@ -28,8 +28,8 @@ import org.jetlinks.supports.official.JetLinksDeviceMetadataCodec; import javax.persistence.Column; import javax.persistence.Index; import javax.persistence.Table; -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.NotNull; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; import java.io.ObjectInput; import java.io.ObjectOutput; import java.util.Date; diff --git a/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/service/data/DeviceDataManagerSupport.java b/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/service/data/DeviceDataManagerSupport.java deleted file mode 100644 index 38a7791a..00000000 --- a/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/service/data/DeviceDataManagerSupport.java +++ /dev/null @@ -1,106 +0,0 @@ -package org.jetlinks.community.device.service.data; - -import lombok.AllArgsConstructor; -import org.hswebframework.web.api.crud.entity.QueryParamEntity; -import org.jetlinks.core.codec.Codec; -import org.jetlinks.core.codec.Codecs; -import org.jetlinks.core.event.EventBus; -import org.jetlinks.core.event.Subscription; -import org.jetlinks.core.message.ThingMessage; -import org.jetlinks.core.message.property.PropertyMessage; -import org.jetlinks.core.things.ThingProperty; -import org.jetlinks.core.things.ThingType; -import org.jetlinks.core.things.ThingsDataManagerSupport; -import org.jetlinks.community.things.ThingConstants; -import org.springframework.stereotype.Component; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; - -@AllArgsConstructor -@Component -public class DeviceDataManagerSupport implements ThingsDataManagerSupport { - private final DeviceDataService dataService; - - private final EventBus eventBus; - - @Override - public boolean isSupported(ThingType thingType) { - return true; - } - - @Override - public Mono getFirstProperty(ThingType thingType, String thingId) { - // FIXME: 2021/11/2 使用缓存 - // order by timestamp asc limit 0,1 - return QueryParamEntity - .newQuery() - .where() - .doPaging(0, 1) - .orderByAsc("timestamp") - .execute(param -> dataService.queryProperty(thingId, param)) - .take(1) - .map(data -> ThingProperty.of(data.getProperty(), data.getValue(), data.getTimestamp(), data.getState())) - .singleOrEmpty(); - } - - @Override - public Mono getAnyLastProperty(ThingType thingType, String thingId, long baseTime) { - // where timestamp < ? order by timestamp desc limit 0,1 - return QueryParamEntity - .newQuery() - .where() - .lt("timestamp", baseTime) - .doPaging(0, 1) - .orderByDesc("timestamp") - .execute(param -> dataService.queryProperty(thingId, param)) - .take(1) - .map(data -> ThingProperty.of(data.getProperty(), data.getValue(), data.getTimestamp(), data.getState())) - .singleOrEmpty(); - } - - @Override - public Mono getLastProperty(ThingType thingType, String thingId, String property, long baseTime) { - // where property = ? and timestamp < ? order by timestamp desc limit 0,1 - return QueryParamEntity - .newQuery() - .where() - .lt("timestamp", baseTime) - .doPaging(0, 1) - .orderByDesc("timestamp") - .execute(param -> dataService.queryProperty(thingId, param, property)) - .take(1) - .map(data -> ThingProperty.of(data.getProperty(), data.getValue(), data.getTimestamp(), data.getState())) - .singleOrEmpty(); - } - - @Override - public Mono getFirstProperty(ThingType thingType, String thingId, String property) { - // where property = ? order by timestamp asc limit 0,1 - return QueryParamEntity - .newQuery() - .where() - .doPaging(0, 1) - .orderByAsc("timestamp") - .execute(param -> dataService.queryProperty(thingId, param, property)) - .take(1) - .map(data -> ThingProperty.of(data.getProperty(), data.getValue(), data.getTimestamp(), data.getState())) - .singleOrEmpty(); - } - - private final static Codec CODEC = Codecs.lookup(ThingMessage.class); - - @Override - public Flux subscribeProperty(ThingType thingType, String thingId) { - return eventBus - .subscribe(Subscription - .builder() - .topics(ThingConstants.Topics.properties(thingType, thingId)) - .subscriberId("device-data:" + thingId) - .local() - .broker() - .build(), - CODEC) - .cast(PropertyMessage.class) - .flatMapIterable(PropertyMessage::getCompleteProperties); - } -} diff --git a/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/service/data/DeviceDataService.java b/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/service/data/DeviceDataService.java index 6403f34b..fc17fc03 100644 --- a/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/service/data/DeviceDataService.java +++ b/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/service/data/DeviceDataService.java @@ -29,8 +29,8 @@ import reactor.core.publisher.Mono; import javax.annotation.Nonnull; import javax.annotation.Nullable; -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.NotNull; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; import java.util.Collection; import java.util.Date; import java.util.List; diff --git a/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/service/data/DeviceLatestDataService.java b/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/service/data/DeviceLatestDataService.java index 2ab06308..663abb9d 100755 --- a/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/service/data/DeviceLatestDataService.java +++ b/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/service/data/DeviceLatestDataService.java @@ -13,8 +13,8 @@ import org.jetlinks.community.timeseries.query.AggregationColumn; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.NotNull; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; import java.util.List; import java.util.Map; diff --git a/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/web/excel/DeviceExcelInfo.java b/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/web/excel/DeviceExcelInfo.java index c5a5588a..a04b714c 100644 --- a/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/web/excel/DeviceExcelInfo.java +++ b/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/web/excel/DeviceExcelInfo.java @@ -22,7 +22,7 @@ import org.jetlinks.community.io.excel.ExcelUtils; import org.springframework.util.ObjectUtils; import org.springframework.util.StringUtils; -import javax.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotBlank; import java.util.*; import java.util.stream.Collectors; diff --git a/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/web/excel/PropertyMetadataExcelImportInfo.java b/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/web/excel/PropertyMetadataExcelImportInfo.java index bdc152dd..20ee14ee 100644 --- a/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/web/excel/PropertyMetadataExcelImportInfo.java +++ b/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/web/excel/PropertyMetadataExcelImportInfo.java @@ -20,8 +20,8 @@ import org.jetlinks.core.metadata.unit.ValueUnit; import org.jetlinks.core.metadata.unit.ValueUnits; import org.jetlinks.supports.official.JetLinksDataTypeCodecs; -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.Pattern; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.Pattern; import java.util.*; import java.util.function.Supplier; import java.util.stream.Collectors; diff --git a/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/web/excel/PropertyMetadataExcelInfo.java b/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/web/excel/PropertyMetadataExcelInfo.java index af7bd960..435a4490 100644 --- a/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/web/excel/PropertyMetadataExcelInfo.java +++ b/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/web/excel/PropertyMetadataExcelInfo.java @@ -22,7 +22,7 @@ import org.jetlinks.core.metadata.unit.ValueUnits; import org.jetlinks.supports.official.JetLinksDataTypeCodecs; import reactor.core.publisher.Flux; -import javax.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotBlank; import java.util.*; import java.util.function.Supplier; import java.util.stream.Collectors; diff --git a/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/web/request/TransparentMessageCodecRequest.java b/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/web/request/TransparentMessageCodecRequest.java index b421929d..11a87a65 100644 --- a/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/web/request/TransparentMessageCodecRequest.java +++ b/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/web/request/TransparentMessageCodecRequest.java @@ -5,7 +5,7 @@ import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; -import javax.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotBlank; import java.util.Map; @Getter diff --git a/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/web/request/TransparentMessageDecodeRequest.java b/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/web/request/TransparentMessageDecodeRequest.java index 8fb15215..17ed7403 100644 --- a/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/web/request/TransparentMessageDecodeRequest.java +++ b/jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/web/request/TransparentMessageDecodeRequest.java @@ -8,7 +8,7 @@ import org.apache.commons.collections4.MapUtils; import org.hswebframework.web.validator.ValidatorUtils; import org.jetlinks.core.message.DirectDeviceMessage; -import javax.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotBlank; import java.nio.charset.StandardCharsets; import java.util.Map; diff --git a/jetlinks-manager/logging-manager/pom.xml b/jetlinks-manager/logging-manager/pom.xml index 95f7c403..685c925b 100644 --- a/jetlinks-manager/logging-manager/pom.xml +++ b/jetlinks-manager/logging-manager/pom.xml @@ -7,7 +7,7 @@ org.jetlinks.community jetlinks-manager - 2.3.0-SNAPSHOT + 2.10.0-SNAPSHOT logging-manager diff --git a/jetlinks-manager/logging-manager/src/main/java/org/jetlinks/community/logging/controller/AccessLoggerController.java b/jetlinks-manager/logging-manager/src/main/java/org/jetlinks/community/logging/controller/AccessLoggerController.java index a49e975d..5f865632 100644 --- a/jetlinks-manager/logging-manager/src/main/java/org/jetlinks/community/logging/controller/AccessLoggerController.java +++ b/jetlinks-manager/logging-manager/src/main/java/org/jetlinks/community/logging/controller/AccessLoggerController.java @@ -3,27 +3,17 @@ package org.jetlinks.community.logging.controller; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.tags.Tag; +import lombok.AllArgsConstructor; import org.hswebframework.web.api.crud.entity.PagerResult; -import org.hswebframework.web.api.crud.entity.QueryNoPagingOperation; import org.hswebframework.web.api.crud.entity.QueryOperation; import org.hswebframework.web.api.crud.entity.QueryParamEntity; -import org.hswebframework.web.authorization.Authentication; import org.hswebframework.web.authorization.annotation.QueryAction; import org.hswebframework.web.authorization.annotation.Resource; +import org.jetlinks.community.logging.access.AccessLoggerService; import org.jetlinks.community.logging.access.SerializableAccessLog; -import org.jetlinks.community.logging.service.AccessLoggerService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.io.buffer.DataBufferFactory; -import org.springframework.core.io.buffer.DefaultDataBufferFactory; -import org.springframework.http.HttpHeaders; -import org.springframework.http.server.reactive.ServerHttpResponse; import org.springframework.web.bind.annotation.*; import reactor.core.publisher.Mono; -import java.io.IOException; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; - /** * @author bsetfeng * @since 1.0 @@ -32,9 +22,9 @@ import java.nio.charset.StandardCharsets; @RequestMapping("/logger/access") @Resource(id = "access-logger", name = "访问日志") @Tag(name = "日志管理") +@AllArgsConstructor public class AccessLoggerController { - @Autowired private AccessLoggerService loggerService; @GetMapping("/_query") @@ -42,7 +32,7 @@ public class AccessLoggerController { @QueryOperation(summary = "查询访问日志") public Mono> getAccessLogger(@Parameter(hidden = true) QueryParamEntity queryParam) { return loggerService - .getAccessLogger(queryParam); + .query(queryParam); } @PostMapping("/_query") @@ -50,7 +40,7 @@ public class AccessLoggerController { @Operation(summary = "(POST)查询访问日志") public Mono> getAccessLogger(@RequestBody Mono queryMono) { return queryMono - .flatMap(loggerService::getAccessLogger); + .flatMap(loggerService::query); } diff --git a/jetlinks-manager/logging-manager/src/main/java/org/jetlinks/community/logging/controller/SystemLoggerController.java b/jetlinks-manager/logging-manager/src/main/java/org/jetlinks/community/logging/controller/SystemLoggerController.java index 6de82fd9..ab107551 100644 --- a/jetlinks-manager/logging-manager/src/main/java/org/jetlinks/community/logging/controller/SystemLoggerController.java +++ b/jetlinks-manager/logging-manager/src/main/java/org/jetlinks/community/logging/controller/SystemLoggerController.java @@ -8,8 +8,8 @@ import org.hswebframework.web.api.crud.entity.QueryOperation; import org.hswebframework.web.api.crud.entity.QueryParamEntity; import org.hswebframework.web.authorization.annotation.QueryAction; import org.hswebframework.web.authorization.annotation.Resource; -import org.jetlinks.community.logging.service.SystemLoggerService; import org.jetlinks.community.logging.system.SerializableSystemLog; +import org.jetlinks.community.logging.system.SystemLoggerService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; import reactor.core.publisher.Mono; @@ -31,7 +31,7 @@ public class SystemLoggerController { @QueryAction @QueryOperation(summary = "查询系统日志") public Mono> getSystemLogger(@Parameter(hidden = true) QueryParamEntity queryParam) { - return loggerService.getSystemLogger(queryParam); + return loggerService.query(queryParam); } @PostMapping("/_query") @@ -39,7 +39,7 @@ public class SystemLoggerController { @Operation(summary = "(POST)查询系统日志") public Mono> getSystemLogger(@RequestBody Mono queryMono) { return queryMono - .flatMap(queryParam -> loggerService.getSystemLogger(queryParam)); + .flatMap(queryParam -> loggerService.query(queryParam)); } diff --git a/jetlinks-manager/logging-manager/src/main/java/org/jetlinks/community/logging/service/AccessLoggerService.java b/jetlinks-manager/logging-manager/src/main/java/org/jetlinks/community/logging/service/AccessLoggerService.java deleted file mode 100644 index 7de1ae1e..00000000 --- a/jetlinks-manager/logging-manager/src/main/java/org/jetlinks/community/logging/service/AccessLoggerService.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.jetlinks.community.logging.service; - - -import org.hswebframework.ezorm.core.param.QueryParam; -import org.hswebframework.web.api.crud.entity.PagerResult; -import org.jetlinks.community.elastic.search.service.ElasticSearchService; -import org.jetlinks.community.logging.access.SerializableAccessLog; -import org.jetlinks.community.logging.event.handler.LoggerIndexProvider; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import reactor.core.publisher.Mono; - -/** - * @version 1.0 - **/ -@Service -public class AccessLoggerService { - - @Autowired - private ElasticSearchService searchService; - - public Mono> getAccessLogger(QueryParam queryParam) { - return searchService.queryPager(LoggerIndexProvider.ACCESS, queryParam, SerializableAccessLog.class); - } - -} diff --git a/jetlinks-manager/logging-manager/src/main/java/org/jetlinks/community/logging/service/SystemLoggerService.java b/jetlinks-manager/logging-manager/src/main/java/org/jetlinks/community/logging/service/SystemLoggerService.java deleted file mode 100644 index aa6b499e..00000000 --- a/jetlinks-manager/logging-manager/src/main/java/org/jetlinks/community/logging/service/SystemLoggerService.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.jetlinks.community.logging.service; - - -import org.hswebframework.ezorm.core.param.QueryParam; -import org.hswebframework.web.api.crud.entity.PagerResult; -import org.jetlinks.community.elastic.search.service.ElasticSearchService; -import org.jetlinks.community.logging.event.handler.LoggerIndexProvider; -import org.jetlinks.community.logging.system.SerializableSystemLog; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import reactor.core.publisher.Mono; - -/** - * @version 1.0 - **/ -@Service -public class SystemLoggerService { - - @Autowired - private ElasticSearchService searchService; - - public Mono> getSystemLogger(QueryParam queryParam) { - return searchService.queryPager(LoggerIndexProvider.SYSTEM, queryParam, SerializableSystemLog.class); - } -} diff --git a/jetlinks-manager/network-manager/pom.xml b/jetlinks-manager/network-manager/pom.xml index f81a6442..c274c86f 100644 --- a/jetlinks-manager/network-manager/pom.xml +++ b/jetlinks-manager/network-manager/pom.xml @@ -7,7 +7,7 @@ org.jetlinks.community jetlinks-manager - 2.3.0-SNAPSHOT + 2.10.0-SNAPSHOT ../pom.xml network-manager diff --git a/jetlinks-manager/network-manager/src/main/java/org/jetlinks/community/network/manager/debug/TcpServerDebugSubscriptionProvider.java b/jetlinks-manager/network-manager/src/main/java/org/jetlinks/community/network/manager/debug/TcpServerDebugSubscriptionProvider.java index 943621c3..df80dcb9 100644 --- a/jetlinks-manager/network-manager/src/main/java/org/jetlinks/community/network/manager/debug/TcpServerDebugSubscriptionProvider.java +++ b/jetlinks-manager/network-manager/src/main/java/org/jetlinks/community/network/manager/debug/TcpServerDebugSubscriptionProvider.java @@ -85,7 +85,7 @@ public class TcpServerDebugSubscriptionProvider implements SubscriptionProvider .then() ) .doOnError(sink::error) - .subscriberContext(sink.currentContext()) + .contextWrite(sink.currentContext()) .subscribe() )); } diff --git a/jetlinks-manager/network-manager/src/main/java/org/jetlinks/community/network/manager/entity/CertificateEntity.java b/jetlinks-manager/network-manager/src/main/java/org/jetlinks/community/network/manager/entity/CertificateEntity.java index b514a6b7..b09b4abb 100755 --- a/jetlinks-manager/network-manager/src/main/java/org/jetlinks/community/network/manager/entity/CertificateEntity.java +++ b/jetlinks-manager/network-manager/src/main/java/org/jetlinks/community/network/manager/entity/CertificateEntity.java @@ -19,7 +19,7 @@ import org.springframework.util.Assert; import javax.persistence.Column; import javax.persistence.Table; -import javax.validation.constraints.NotNull; +import jakarta.validation.constraints.NotNull; import java.sql.JDBCType; /** diff --git a/jetlinks-manager/network-manager/src/main/java/org/jetlinks/community/network/manager/entity/DeviceGatewayEntity.java b/jetlinks-manager/network-manager/src/main/java/org/jetlinks/community/network/manager/entity/DeviceGatewayEntity.java index 85388292..f1e8f7eb 100755 --- a/jetlinks-manager/network-manager/src/main/java/org/jetlinks/community/network/manager/entity/DeviceGatewayEntity.java +++ b/jetlinks-manager/network-manager/src/main/java/org/jetlinks/community/network/manager/entity/DeviceGatewayEntity.java @@ -19,7 +19,7 @@ import org.jetlinks.core.ProtocolSupport; import javax.persistence.Column; import javax.persistence.Table; -import javax.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotBlank; import java.sql.JDBCType; import java.util.Map; diff --git a/jetlinks-manager/network-manager/src/main/java/org/jetlinks/community/network/manager/entity/NetworkConfigEntity.java b/jetlinks-manager/network-manager/src/main/java/org/jetlinks/community/network/manager/entity/NetworkConfigEntity.java index 53b7b0c7..2721c8db 100755 --- a/jetlinks-manager/network-manager/src/main/java/org/jetlinks/community/network/manager/entity/NetworkConfigEntity.java +++ b/jetlinks-manager/network-manager/src/main/java/org/jetlinks/community/network/manager/entity/NetworkConfigEntity.java @@ -17,7 +17,7 @@ import org.jetlinks.community.network.manager.enums.NetworkConfigState; import javax.persistence.Column; import javax.persistence.Table; -import javax.validation.constraints.NotNull; +import jakarta.validation.constraints.NotNull; import java.io.Serializable; import java.sql.JDBCType; import java.util.*; diff --git a/jetlinks-manager/notify-manager/pom.xml b/jetlinks-manager/notify-manager/pom.xml index 3166d9e3..71b51599 100644 --- a/jetlinks-manager/notify-manager/pom.xml +++ b/jetlinks-manager/notify-manager/pom.xml @@ -7,7 +7,7 @@ org.jetlinks.community jetlinks-manager - 2.3.0-SNAPSHOT + 2.10.0-SNAPSHOT ../pom.xml notify-manager @@ -98,7 +98,7 @@ org.jetlinks.community - elasticsearch-component + timeseries-component ${project.version} compile diff --git a/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/configuration/ElasticSearchNotifyHistoryConfiguration.java b/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/configuration/ElasticSearchNotifyHistoryConfiguration.java deleted file mode 100644 index a3fdffe1..00000000 --- a/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/configuration/ElasticSearchNotifyHistoryConfiguration.java +++ /dev/null @@ -1,24 +0,0 @@ -package org.jetlinks.community.notify.manager.configuration; - -import org.jetlinks.community.elastic.search.configuration.ElasticSearchConfiguration; -import org.jetlinks.community.elastic.search.index.ElasticSearchIndexManager; -import org.jetlinks.community.elastic.search.service.ElasticSearchService; -import org.jetlinks.community.notify.manager.service.ElasticSearchNotifyHistoryRepository; -import org.springframework.boot.autoconfigure.AutoConfiguration; -import org.springframework.boot.autoconfigure.AutoConfigureAfter; -import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; -import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; -import org.springframework.context.annotation.Bean; - -@AutoConfiguration -@ConditionalOnClass(ElasticSearchService.class) -@AutoConfigureAfter(ElasticSearchConfiguration.class) -public class ElasticSearchNotifyHistoryConfiguration { - - @Bean(initMethod = "init") - @ConditionalOnBean(ElasticSearchService.class) - public ElasticSearchNotifyHistoryRepository notifyHistoryRepository(ElasticSearchService elasticSearchService, - ElasticSearchIndexManager indexManager) { - return new ElasticSearchNotifyHistoryRepository(elasticSearchService, indexManager); - } -} \ No newline at end of file diff --git a/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/configuration/NotifyConfiguration.java b/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/configuration/NotifyConfiguration.java index 6e657c0b..24cbbea9 100644 --- a/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/configuration/NotifyConfiguration.java +++ b/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/configuration/NotifyConfiguration.java @@ -1,21 +1,18 @@ package org.jetlinks.community.notify.manager.configuration; -import org.jetlinks.community.notify.manager.service.InDatabaseNotifyHistoryRepository; -import org.jetlinks.community.notify.manager.service.NotifyHistoryRepository; -import org.jetlinks.community.notify.manager.service.NotifyHistoryService; +import org.jetlinks.community.notify.manager.service.TimeSeriesNotifyHistoryRepository; +import org.jetlinks.community.timeseries.TimeSeriesManager; import org.springframework.boot.autoconfigure.AutoConfiguration; -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; @AutoConfiguration @EnableConfigurationProperties({NotifySubscriberProperties.class, NotificationProperties.class}) public class NotifyConfiguration { @Bean - @ConditionalOnMissingBean(NotifyHistoryRepository.class) - public NotifyHistoryRepository notifyHistoryRepository(NotifyHistoryService historyService) { - return new InDatabaseNotifyHistoryRepository(historyService); + public TimeSeriesNotifyHistoryRepository notifyHistoryRepository(TimeSeriesManager timeSeriesManager) { + return new TimeSeriesNotifyHistoryRepository(timeSeriesManager); } + } diff --git a/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/entity/NotifyConfigEntity.java b/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/entity/NotifyConfigEntity.java index e9792d41..df85c533 100755 --- a/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/entity/NotifyConfigEntity.java +++ b/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/entity/NotifyConfigEntity.java @@ -18,8 +18,8 @@ import org.jetlinks.community.notify.NotifierProperties; import javax.persistence.Column; import javax.persistence.GeneratedValue; import javax.persistence.Table; -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.Pattern; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.Pattern; import java.sql.JDBCType; import java.util.HashMap; import java.util.List; diff --git a/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/entity/NotifySubscriberChannelEntity.java b/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/entity/NotifySubscriberChannelEntity.java index aec5fa93..238d8d0d 100644 --- a/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/entity/NotifySubscriberChannelEntity.java +++ b/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/entity/NotifySubscriberChannelEntity.java @@ -18,7 +18,7 @@ import org.jetlinks.community.notify.manager.subscriber.channel.NotifyChannelPro import javax.persistence.Column; import javax.persistence.Table; -import javax.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotBlank; import java.sql.JDBCType; import java.util.Map; diff --git a/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/entity/NotifySubscriberProviderEntity.java b/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/entity/NotifySubscriberProviderEntity.java index 62e8f68f..b2f38940 100644 --- a/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/entity/NotifySubscriberProviderEntity.java +++ b/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/entity/NotifySubscriberProviderEntity.java @@ -17,7 +17,7 @@ import org.jetlinks.community.notify.manager.enums.NotifyChannelState; import javax.persistence.Column; import javax.persistence.Table; -import javax.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotBlank; import java.sql.JDBCType; import java.util.Map; diff --git a/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/entity/NotifyTemplateEntity.java b/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/entity/NotifyTemplateEntity.java index 00a8548c..25af5a48 100755 --- a/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/entity/NotifyTemplateEntity.java +++ b/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/entity/NotifyTemplateEntity.java @@ -18,8 +18,8 @@ import org.jetlinks.community.notify.template.VariableDefinition; import javax.persistence.Column; import javax.persistence.GeneratedValue; import javax.persistence.Table; -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.Pattern; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.Pattern; import java.sql.JDBCType; import java.util.List; import java.util.Map; diff --git a/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/service/DefaultTemplateManager.java b/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/service/DefaultTemplateManager.java index f9db4f6e..2873c1b9 100644 --- a/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/service/DefaultTemplateManager.java +++ b/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/service/DefaultTemplateManager.java @@ -8,32 +8,46 @@ import org.jetlinks.community.notify.template.TemplateProperties; import org.jetlinks.community.notify.template.TemplateProvider; import org.jetlinks.core.event.EventBus; import org.springframework.beans.BeansException; +import org.springframework.beans.factory.SmartInitializingSingleton; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.config.BeanPostProcessor; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; import org.springframework.stereotype.Service; import reactor.core.publisher.Mono; +import java.util.Collections; +import java.util.List; + @Service @Slf4j -public class DefaultTemplateManager extends AbstractTemplateManager implements BeanPostProcessor { +public class DefaultTemplateManager extends AbstractTemplateManager + implements SmartInitializingSingleton { private final NotifyTemplateService templateService; - public DefaultTemplateManager(EventBus eventBus, NotifyTemplateService templateService) { + private final ApplicationContext context; + + @Autowired + public DefaultTemplateManager(EventBus eventBus, NotifyTemplateService templateService, ApplicationContext context) { super(eventBus); this.templateService = templateService; + this.context = context; } @Override protected Mono getProperties(NotifyType type, String id) { - return templateService.findById(Mono.just(id)) - .map(NotifyTemplateEntity::toTemplateProperties); - } - @Override - public Object postProcessAfterInitialization(Object bean, String beanName) throws BeansException { - if (bean instanceof TemplateProvider) { - register(((TemplateProvider) bean)); - } - return bean; + return templateService + .findById(Mono.just(id)) + .map(NotifyTemplateEntity::toTemplateProperties); } + @Override + public void afterSingletonsInstantiated() { + context + .getBeanProvider(TemplateProvider.class) + .forEach(this::register); + } + + } diff --git a/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/service/ElasticSearchNotifyHistoryRepository.java b/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/service/ElasticSearchNotifyHistoryRepository.java deleted file mode 100644 index 9423b8b9..00000000 --- a/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/service/ElasticSearchNotifyHistoryRepository.java +++ /dev/null @@ -1,61 +0,0 @@ -package org.jetlinks.community.notify.manager.service; - -import com.alibaba.fastjson.JSON; -import lombok.AllArgsConstructor; -import org.hswebframework.web.api.crud.entity.PagerResult; -import org.hswebframework.web.api.crud.entity.QueryParamEntity; -import org.hswebframework.web.bean.FastBeanCopier; -import org.jetlinks.core.metadata.types.DateTimeType; -import org.jetlinks.core.metadata.types.StringType; -import org.jetlinks.community.elastic.search.index.DefaultElasticSearchIndexMetadata; -import org.jetlinks.community.elastic.search.index.ElasticSearchIndexManager; -import org.jetlinks.community.elastic.search.service.ElasticSearchService; -import org.jetlinks.community.gateway.annotation.Subscribe; -import org.jetlinks.community.notify.event.SerializableNotifierEvent; -import org.springframework.stereotype.Component; -import org.springframework.transaction.annotation.Propagation; -import org.springframework.transaction.annotation.Transactional; -import reactor.core.publisher.Mono; - -import javax.annotation.PostConstruct; -import java.time.Duration; - -@AllArgsConstructor -@Component -public class ElasticSearchNotifyHistoryRepository implements NotifyHistoryRepository { - - public static final String INDEX_NAME = "notify_history"; - - private final ElasticSearchService elasticSearchService; - private final ElasticSearchIndexManager indexManager; - - @PostConstruct - public void init() { - indexManager - .putIndex( - new DefaultElasticSearchIndexMetadata(INDEX_NAME) - .addProperty("id", StringType.GLOBAL) - .addProperty("notifyTime", DateTimeType.GLOBAL) - //其他字段默认都是string - ) - .block(Duration.ofSeconds(10)); - } - - @Subscribe("/notify/**") - @Transactional(propagation = Propagation.NEVER) - public Mono handleNotify(SerializableNotifierEvent event) { - return elasticSearchService - .commit(INDEX_NAME, NotifyHistory.of(event).toJson()); - } - - - @Override - public Mono> queryPager(QueryParamEntity param) { - return elasticSearchService - .queryPager(INDEX_NAME, param, map -> { - NotifyHistory history = FastBeanCopier.copy(map, new NotifyHistory(),"context"); - history.setContext(JSON.parseObject((String) map.getOrDefault("context", "{}"))); - return history; - }); - } -} diff --git a/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/service/InDatabaseNotifyHistoryRepository.java b/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/service/InDatabaseNotifyHistoryRepository.java deleted file mode 100644 index c196ae14..00000000 --- a/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/service/InDatabaseNotifyHistoryRepository.java +++ /dev/null @@ -1,32 +0,0 @@ -package org.jetlinks.community.notify.manager.service; - -import lombok.AllArgsConstructor; -import org.hswebframework.web.api.crud.entity.PagerResult; -import org.hswebframework.web.api.crud.entity.QueryParamEntity; -import org.jetlinks.community.gateway.annotation.Subscribe; -import org.jetlinks.community.notify.event.SerializableNotifierEvent; -import org.jetlinks.community.notify.manager.entity.NotifyHistoryEntity; -import org.springframework.transaction.annotation.Propagation; -import org.springframework.transaction.annotation.Transactional; -import reactor.core.publisher.Mono; - -@AllArgsConstructor -public class InDatabaseNotifyHistoryRepository implements NotifyHistoryRepository { - - private final NotifyHistoryService historyService; - - @Subscribe("/notify/**") - @Transactional(propagation = Propagation.NEVER) - public Mono handleNotify(SerializableNotifierEvent event) { - - return historyService - .save(NotifyHistoryEntity.of(event)) - .then(); - } - - @Override - public Mono> queryPager(QueryParamEntity param) { - return historyService - .queryPager(param, NotifyHistoryEntity::toHistory); - } -} diff --git a/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/service/TimeSeriesNotifyHistoryRepository.java b/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/service/TimeSeriesNotifyHistoryRepository.java new file mode 100644 index 00000000..7443a1d9 --- /dev/null +++ b/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/service/TimeSeriesNotifyHistoryRepository.java @@ -0,0 +1,78 @@ +package org.jetlinks.community.notify.manager.service; + +import com.alibaba.fastjson.JSON; +import lombok.AllArgsConstructor; +import org.hswebframework.web.api.crud.entity.PagerResult; +import org.hswebframework.web.api.crud.entity.QueryParamEntity; +import org.hswebframework.web.bean.FastBeanCopier; +import org.jetlinks.core.metadata.types.DateTimeType; +import org.jetlinks.core.metadata.types.StringType; +import org.jetlinks.community.ConfigMetadataConstants; +import org.jetlinks.community.gateway.annotation.Subscribe; +import org.jetlinks.community.notify.event.SerializableNotifierEvent; +import org.jetlinks.community.timeseries.TimeSeriesData; +import org.jetlinks.community.timeseries.TimeSeriesManager; +import org.jetlinks.community.timeseries.TimeSeriesMetadata; +import org.jetlinks.community.timeseries.TimeSeriesMetric; +import org.springframework.beans.factory.SmartInitializingSingleton; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; +import reactor.core.publisher.Mono; + +import java.time.Duration; + +import static org.jetlinks.core.metadata.SimplePropertyMetadata.*; + +@AllArgsConstructor +public class TimeSeriesNotifyHistoryRepository implements NotifyHistoryRepository, SmartInitializingSingleton { + + public static final TimeSeriesMetric METRIC = TimeSeriesMetric.of("notify_history"); + + private final TimeSeriesManager timeSeriesManager; + + @Subscribe("/notify/**") + @Transactional(propagation = Propagation.NEVER) + public Mono handleNotify(SerializableNotifierEvent event) { + return timeSeriesManager + .getService(METRIC) + .commit(TimeSeriesData.of(event.getSendTime(), NotifyHistory.of(event).toJson())); + } + + + @Override + public Mono> queryPager(QueryParamEntity param) { + return timeSeriesManager + .getService(METRIC) + .queryPager(param, map -> { + NotifyHistory history = FastBeanCopier.copy(map.getData(), new NotifyHistory(), "context"); + history.setContext(JSON.parseObject((String) map.get("context").orElse("{}"))); + return history; + }); + } + + @Override + public void afterSingletonsInstantiated() { + + timeSeriesManager + .registerMetadata( + TimeSeriesMetadata.of( + METRIC, + of("id", "ID", StringType.GLOBAL), + of("provider", "通知提供商", StringType.GLOBAL), + of("notifyType", "通知类型", StringType.GLOBAL), + of("notifierId", "通知器ID", StringType.GLOBAL), + of("state", "通知状态", StringType.GLOBAL), + of("notifyTime", "通知时间", DateTimeType.GLOBAL), + of("context", "通知上下文", new StringType() + .expand(ConfigMetadataConstants.maxLength, 8096L)), + of("template", "模版内容", new StringType() + .expand(ConfigMetadataConstants.maxLength, 8096L)), + of("errorType", "错误类型", StringType.GLOBAL), + of("errorStack", "异常栈", new StringType() + .expand(ConfigMetadataConstants.maxLength, 8096L)), + of("templateId", "模版ID", StringType.GLOBAL) + ) + ) + .block(Duration.ofSeconds(10)); + } +} diff --git a/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/subscriber/channel/notifiers/NotifierChannelProvider.java b/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/subscriber/channel/notifiers/NotifierChannelProvider.java index 6b0990a0..c65b4faa 100644 --- a/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/subscriber/channel/notifiers/NotifierChannelProvider.java +++ b/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/subscriber/channel/notifiers/NotifierChannelProvider.java @@ -16,7 +16,7 @@ import org.jetlinks.community.notify.manager.subscriber.channel.NotifyChannel; import org.jetlinks.community.notify.manager.subscriber.channel.NotifyChannelProvider; import reactor.core.publisher.Mono; -import javax.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotBlank; import java.util.Map; @AllArgsConstructor diff --git a/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/web/NotifierController.java b/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/web/NotifierController.java index 992cfbcc..fe601cb3 100755 --- a/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/web/NotifierController.java +++ b/jetlinks-manager/notify-manager/src/main/java/org/jetlinks/community/notify/manager/web/NotifierController.java @@ -19,7 +19,7 @@ import org.jetlinks.core.Values; import org.springframework.web.bind.annotation.*; import reactor.core.publisher.Mono; -import javax.validation.constraints.NotNull; +import jakarta.validation.constraints.NotNull; import java.util.HashMap; import java.util.Map; import java.util.function.Function; diff --git a/jetlinks-manager/pom.xml b/jetlinks-manager/pom.xml index 787ea06d..0229611b 100644 --- a/jetlinks-manager/pom.xml +++ b/jetlinks-manager/pom.xml @@ -5,7 +5,7 @@ jetlinks-community org.jetlinks.community - 2.3.0-SNAPSHOT + 2.10.0-SNAPSHOT pom 4.0.0 diff --git a/jetlinks-manager/rule-engine-manager/pom.xml b/jetlinks-manager/rule-engine-manager/pom.xml index 72dff48d..b4c9ae3e 100644 --- a/jetlinks-manager/rule-engine-manager/pom.xml +++ b/jetlinks-manager/rule-engine-manager/pom.xml @@ -7,7 +7,7 @@ org.jetlinks.community jetlinks-manager - 2.3.0-SNAPSHOT + 2.10.0-SNAPSHOT ../pom.xml rule-engine-manager @@ -40,7 +40,7 @@ ${project.groupId} - elasticsearch-component + timeseries-component ${project.version} diff --git a/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/alarm/AlarmHandleInfo.java b/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/alarm/AlarmHandleInfo.java index 73c28266..5e80c481 100644 --- a/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/alarm/AlarmHandleInfo.java +++ b/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/alarm/AlarmHandleInfo.java @@ -7,8 +7,8 @@ import org.jetlinks.community.rule.engine.enums.AlarmHandleState; import org.jetlinks.community.rule.engine.enums.AlarmRecordState; import org.jetlinks.community.terms.TermSpec; -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.NotNull; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; /** * @author bestfeng diff --git a/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/alarm/AlarmTaskExecutorProvider.java b/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/alarm/AlarmTaskExecutorProvider.java index 397d5a9f..8adc0425 100644 --- a/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/alarm/AlarmTaskExecutorProvider.java +++ b/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/alarm/AlarmTaskExecutorProvider.java @@ -17,7 +17,7 @@ import org.reactivestreams.Publisher; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -import javax.validation.constraints.NotNull; +import jakarta.validation.constraints.NotNull; import java.io.Serializable; import java.util.function.Function; diff --git a/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/configuration/RuleEngineManagerConfiguration.java b/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/configuration/RuleEngineManagerConfiguration.java index bd2ca623..7ad90183 100644 --- a/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/configuration/RuleEngineManagerConfiguration.java +++ b/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/configuration/RuleEngineManagerConfiguration.java @@ -1,15 +1,13 @@ package org.jetlinks.community.rule.engine.configuration; import org.hswebframework.ezorm.rdb.mapping.ReactiveRepository; -import org.jetlinks.community.elastic.search.index.ElasticSearchIndexManager; -import org.jetlinks.community.elastic.search.service.AggregationService; -import org.jetlinks.community.elastic.search.service.ElasticSearchService; import org.jetlinks.community.rule.engine.alarm.AlarmHandler; import org.jetlinks.community.rule.engine.cmd.*; import org.jetlinks.community.rule.engine.entity.AlarmLevelEntity; import org.jetlinks.community.rule.engine.entity.AlarmRuleBindEntity; import org.jetlinks.community.rule.engine.scene.*; import org.jetlinks.community.rule.engine.service.*; +import org.jetlinks.community.timeseries.TimeSeriesManager; import org.jetlinks.core.event.EventBus; import org.springframework.beans.factory.ObjectProvider; import org.springframework.boot.autoconfigure.AutoConfiguration; @@ -51,15 +49,8 @@ public class RuleEngineManagerConfiguration { ); } - @Configuration(proxyBeanMethods = false) - @ConditionalOnClass(ElasticSearchService.class) - static class ElasticSearchAlarmHistoryConfiguration { - - @Bean(initMethod = "init") - public ElasticSearchAlarmHistoryService alarmHistoryService(ElasticSearchService elasticSearchService, - ElasticSearchIndexManager indexManager, - AggregationService aggregationService) { - return new ElasticSearchAlarmHistoryService(indexManager, elasticSearchService, aggregationService); - } + @Bean(initMethod = "init") + public TimeSeriesAlarmHistoryService alarmHistoryService(TimeSeriesManager timeSeriesManager) { + return new TimeSeriesAlarmHistoryService(timeSeriesManager); } } diff --git a/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/entity/AlarmConfigDetail.java b/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/entity/AlarmConfigDetail.java index 38ac3dc2..5764f762 100644 --- a/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/entity/AlarmConfigDetail.java +++ b/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/entity/AlarmConfigDetail.java @@ -9,7 +9,7 @@ import org.jetlinks.community.rule.engine.enums.RuleInstanceState; import org.jetlinks.community.rule.engine.scene.internal.triggers.ManualTriggerProvider; import javax.persistence.Column; -import javax.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotBlank; import java.util.ArrayList; import java.util.List; diff --git a/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/entity/AlarmRuleBindEntity.java b/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/entity/AlarmRuleBindEntity.java index 24c16fb2..e2d877f3 100644 --- a/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/entity/AlarmRuleBindEntity.java +++ b/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/entity/AlarmRuleBindEntity.java @@ -12,7 +12,7 @@ import org.springframework.util.StringUtils; import javax.persistence.Column; import javax.persistence.Index; import javax.persistence.Table; -import javax.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotBlank; @Table(name = "s_alarm_rule_bind", indexes = { @Index(name = "idx_alarm_rule_aid", columnList = "alarmId"), diff --git a/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/entity/RuleInstanceEntity.java b/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/entity/RuleInstanceEntity.java index 865752ed..635fe786 100755 --- a/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/entity/RuleInstanceEntity.java +++ b/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/entity/RuleInstanceEntity.java @@ -24,7 +24,7 @@ import org.springframework.util.StringUtils; import javax.persistence.Column; import javax.persistence.GeneratedValue; import javax.persistence.Table; -import javax.validation.constraints.Pattern; +import jakarta.validation.constraints.Pattern; import java.sql.JDBCType; @Getter diff --git a/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/entity/SceneEntity.java b/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/entity/SceneEntity.java index 10d71bc9..62ccef40 100644 --- a/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/entity/SceneEntity.java +++ b/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/entity/SceneEntity.java @@ -27,8 +27,8 @@ import reactor.core.publisher.Mono; import javax.persistence.Column; import javax.persistence.Table; -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.NotNull; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; import java.sql.JDBCType; import java.util.List; import java.util.Map; diff --git a/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/enums/SqlRuleType.java b/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/enums/SqlRuleType.java deleted file mode 100644 index 608d40d3..00000000 --- a/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/enums/SqlRuleType.java +++ /dev/null @@ -1,65 +0,0 @@ -package org.jetlinks.community.rule.engine.enums; - -import com.alibaba.fastjson.annotation.JSONType; -import lombok.AllArgsConstructor; -import lombok.Getter; -import net.sf.jsqlparser.expression.Expression; -import net.sf.jsqlparser.expression.Function; -import net.sf.jsqlparser.parser.CCJSqlParserUtil; -import net.sf.jsqlparser.statement.select.PlainSelect; -import net.sf.jsqlparser.statement.select.Select; -import org.hswebframework.web.dict.EnumDict; -import org.jetlinks.community.rule.engine.ql.SqlRule; -import org.springframework.scheduling.support.CronSequenceGenerator; -import org.springframework.util.Assert; - -@Getter -@AllArgsConstructor -@JSONType(deserializer = EnumDict.EnumDictJSONDeserializer.class) -public enum SqlRuleType implements EnumDict { - - timer("定时") { - @Override - public void validate(SqlRule rule) { - Assert.notNull(rule.getCron(), "cron表达式不能为空"); - try { - new CronSequenceGenerator(rule.getCron()); - } catch (Exception e) { - throw new IllegalArgumentException("cron表达式格式错误", e); - } - } - }, - realTime("实时") { - @Override - public void validate(SqlRule rule) { - Assert.notNull(rule.getSql(), "sql不能为空"); - try { - PlainSelect select = ((PlainSelect) ((Select) CCJSqlParserUtil.parse(rule.getSql())).getSelectBody()); - if (select.getGroupBy() != null && select.getGroupBy().getGroupByExpressions() != null) { - for (Expression groupByExpression : select.getGroupBy().getGroupByExpressions()) { - if (groupByExpression instanceof Function) { - String name = ((Function) groupByExpression).getName(); - if ("interval".equalsIgnoreCase(name) || "_window".equalsIgnoreCase(name)) { - return; - } - } - } - throw new IllegalArgumentException("实时数据处理必须指定分组函数interval或者_window"); - } - } catch (IllegalArgumentException e) { - throw e; - } catch (Exception e) { - throw new IllegalArgumentException("sql格式错误", e); - } - } - }; - - private final String text; - - @Override - public String getValue() { - return name(); - } - - public abstract void validate(SqlRule rule); -} diff --git a/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/model/Action.java b/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/model/Action.java deleted file mode 100644 index e4fb9604..00000000 --- a/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/model/Action.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.jetlinks.community.rule.engine.model; - -import lombok.Getter; -import lombok.Setter; -import org.jetlinks.rule.engine.api.model.RuleNodeModel; - -import java.io.Serializable; -import java.util.Map; - -@Getter -@Setter -public class Action implements Serializable { - private static final long serialVersionUID = -6849794470754667710L; - - /** - * 执行器 - * - * @see RuleNodeModel#getExecutor() - */ - private String executor; - - /** - * 执行器配置 - * - * @see RuleNodeModel#getConfiguration() - */ - private Map configuration; -} diff --git a/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/model/SqlRuleModelParser.java b/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/model/SqlRuleModelParser.java deleted file mode 100644 index 19f09b29..00000000 --- a/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/model/SqlRuleModelParser.java +++ /dev/null @@ -1,118 +0,0 @@ -package org.jetlinks.community.rule.engine.model; - -import com.alibaba.fastjson.JSON; -import org.jetlinks.community.rule.engine.enums.SqlRuleType; -import org.jetlinks.community.rule.engine.ql.SqlRule; -import org.jetlinks.rule.engine.api.RuleConstants; -import org.jetlinks.rule.engine.api.model.RuleLink; -import org.jetlinks.rule.engine.api.model.RuleModel; -import org.jetlinks.rule.engine.api.model.RuleNodeModel; -import org.jetlinks.rule.engine.model.RuleModelParserStrategy; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; -import org.springframework.util.StringUtils; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -@Component -public class SqlRuleModelParser implements RuleModelParserStrategy { - @Override - public String getFormat() { - return "sql_rule"; - } - - @Override - public RuleModel parse(String modelDefineString) { - - SqlRule sqlRule = JSON.parseObject(modelDefineString, SqlRule.class); - - sqlRule.validate(); - - RuleModel model = new RuleModel(); - model.setId(sqlRule.getId()); - model.setName(sqlRule.getName()); - - RuleNodeModel sqlNode = new RuleNodeModel(); - sqlNode.setId("sql"); - sqlNode.setExecutor("reactor-ql"); - sqlNode.setConfiguration(Collections.singletonMap("sql", sqlRule.getSql())); - sqlNode.setName("SQL"); - - model.getNodes().add(sqlNode); - - //错误处理 - List errorHandler = new ArrayList<>(); - if (!CollectionUtils.isEmpty(sqlRule.getWhenErrorThen())) { - int index = 0; - for (Action act : sqlRule.getWhenErrorThen()) { - if (!StringUtils.hasText(act.getExecutor())) { - continue; - } - index++; - RuleNodeModel action = new RuleNodeModel(); - action.setId("error:action:" + index); - action.setName("错误处理:" + index); - action.setExecutor(act.getExecutor()); - action.setConfiguration(act.getConfiguration()); - RuleLink link = new RuleLink(); - link.setId(action.getId().concat(":").concat(action.getId())); - link.setName("错误处理:" + index); - link.setSource(sqlNode); - link.setType(RuleConstants.Event.error); - link.setTarget(action); - errorHandler.add(link); - model.getNodes().add(action); - } - } - - sqlNode.getEvents().addAll(errorHandler); - - //定时触发 - if (sqlRule.getType() == SqlRuleType.timer) { - RuleNodeModel timerNode = new RuleNodeModel(); - timerNode.setId("timer"); - timerNode.setExecutor("timer"); - timerNode.setName("定时触发"); - timerNode.setConfiguration(Collections.singletonMap("cron", sqlRule.getCron())); - timerNode.setRuleId(model.getId()); - RuleLink link = new RuleLink(); - link.setId("sql:timer"); - link.setName("定时触发SQL"); - link.setSource(timerNode); - link.setTarget(sqlNode); - timerNode.getOutputs().add(link); - sqlNode.getInputs().add(link); - model.getNodes().add(timerNode); - } - - - if (!CollectionUtils.isEmpty(sqlRule.getActions())) { - int index = 0; - for (Action operation : sqlRule.getActions()) { - if (!StringUtils.hasText(operation.getExecutor())) { - continue; - } - index++; - RuleNodeModel action = new RuleNodeModel(); - action.setId("action:" + index); - action.setName("执行动作:" + index); - action.setExecutor(operation.getExecutor()); - action.setConfiguration(operation.getConfiguration()); - RuleLink link = new RuleLink(); - link.setId(action.getId().concat(":").concat(action.getId())); - link.setName("执行动作:" + index); - link.setSource(sqlNode); - link.setTarget(action); - model.getNodes().add(action); - action.getInputs().add(link); - sqlNode.getOutputs().add(link); - - action.getEvents().addAll(errorHandler); - } - } - - return model; - } -} diff --git a/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/ql/SqlRule.java b/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/ql/SqlRule.java deleted file mode 100644 index 7eb9ca1c..00000000 --- a/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/ql/SqlRule.java +++ /dev/null @@ -1,43 +0,0 @@ -package org.jetlinks.community.rule.engine.ql; - -import lombok.Getter; -import lombok.Setter; -import org.jetlinks.community.rule.engine.enums.SqlRuleType; -import org.jetlinks.community.rule.engine.model.Action; -import org.springframework.util.Assert; - -import java.io.Serializable; -import java.util.List; - -/** - * 使用SQL来处理数据 - * - * @author zhouhao - * @since 1.1 - */ -@Getter -@Setter -public class SqlRule implements Serializable { - - private static final long serialVersionUID = -6849794470754667710L; - - private String id; - - private String name; - - private SqlRuleType type; - - private String cron; - - private String sql; - - private List actions; - - private List whenErrorThen; - - public void validate() { - Assert.notNull(type, "type不能为空"); - - type.validate(this); - } -} diff --git a/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/scene/DeviceOperation.java b/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/scene/DeviceOperation.java index 6771e0ee..1480851d 100644 --- a/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/scene/DeviceOperation.java +++ b/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/scene/DeviceOperation.java @@ -20,7 +20,7 @@ import org.jetlinks.community.rule.engine.scene.term.TermColumn; import org.jetlinks.community.rule.engine.utils.TermColumnUtils; import org.springframework.util.Assert; -import javax.validation.constraints.NotNull; +import jakarta.validation.constraints.NotNull; import java.util.ArrayList; import java.util.Collections; import java.util.List; diff --git a/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/scene/SceneAction.java b/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/scene/SceneAction.java index 6dab1303..459fed44 100644 --- a/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/scene/SceneAction.java +++ b/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/scene/SceneAction.java @@ -20,7 +20,7 @@ import org.jetlinks.core.metadata.types.ObjectType; import org.jetlinks.rule.engine.api.model.RuleNodeModel; import reactor.core.publisher.Flux; -import javax.validation.constraints.NotNull; +import jakarta.validation.constraints.NotNull; import java.io.Serializable; import java.util.ArrayList; import java.util.Collections; diff --git a/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/scene/SceneRule.java b/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/scene/SceneRule.java index 93cfddac..53172c3b 100644 --- a/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/scene/SceneRule.java +++ b/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/scene/SceneRule.java @@ -44,8 +44,8 @@ import reactor.core.publisher.Sinks; import reactor.function.Function3; import reactor.util.concurrent.Queues; -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.NotNull; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; import java.io.Serializable; import java.util.*; import java.util.function.Function; diff --git a/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/scene/Trigger.java b/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/scene/Trigger.java index 7aa7a0e2..8df36717 100644 --- a/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/scene/Trigger.java +++ b/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/scene/Trigger.java @@ -23,7 +23,7 @@ import org.springframework.util.CollectionUtils; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -import javax.validation.constraints.NotNull; +import jakarta.validation.constraints.NotNull; import java.io.Serializable; import java.util.ArrayList; import java.util.Collections; diff --git a/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/scene/internal/actions/NotifyAction.java b/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/scene/internal/actions/NotifyAction.java index 358ff601..7fac521b 100644 --- a/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/scene/internal/actions/NotifyAction.java +++ b/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/scene/internal/actions/NotifyAction.java @@ -6,7 +6,7 @@ import lombok.Setter; import org.apache.commons.collections4.MapUtils; import org.jetlinks.community.relation.utils.VariableSource; -import javax.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotBlank; import java.io.Serializable; import java.util.Collections; import java.util.List; diff --git a/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/scene/internal/triggers/DeviceTrigger.java b/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/scene/internal/triggers/DeviceTrigger.java index 22352315..7eac2109 100644 --- a/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/scene/internal/triggers/DeviceTrigger.java +++ b/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/scene/internal/triggers/DeviceTrigger.java @@ -35,8 +35,8 @@ import org.springframework.util.StringUtils; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.NotNull; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; import java.io.Serializable; import java.util.*; import java.util.function.Function; diff --git a/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/service/ElasticSearchAlarmHistoryService.java b/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/service/ElasticSearchAlarmHistoryService.java deleted file mode 100644 index 245cf71b..00000000 --- a/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/service/ElasticSearchAlarmHistoryService.java +++ /dev/null @@ -1,109 +0,0 @@ -package org.jetlinks.community.rule.engine.service; - -import com.alibaba.fastjson.JSONObject; -import lombok.AllArgsConstructor; -import org.hswebframework.ezorm.core.param.QueryParam; -import org.hswebframework.web.api.crud.entity.PagerResult; -import org.hswebframework.web.bean.FastBeanCopier; -import org.jetlinks.community.elastic.search.index.DefaultElasticSearchIndexMetadata; -import org.jetlinks.community.elastic.search.index.ElasticSearchIndexManager; -import org.jetlinks.community.elastic.search.service.AggregationService; -import org.jetlinks.community.elastic.search.service.ElasticSearchService; -import org.jetlinks.community.rule.engine.entity.AlarmHistoryInfo; -import org.jetlinks.community.timeseries.query.AggregationData; -import org.jetlinks.community.timeseries.query.AggregationQueryParam; -import org.jetlinks.core.metadata.types.ArrayType; -import org.jetlinks.core.metadata.types.DateTimeType; -import org.jetlinks.core.metadata.types.IntType; -import org.jetlinks.core.metadata.types.StringType; -import org.springframework.stereotype.Component; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; - -import java.time.Duration; -import java.util.HashMap; -import java.util.Map; - -/** - * @author bestfeng - */ -@AllArgsConstructor -public class ElasticSearchAlarmHistoryService implements AlarmHistoryService { - - - public final static String ALARM_HISTORY_INDEX = "alarm_history"; - - private final ElasticSearchIndexManager indexManager; - - private final ElasticSearchService elasticSearchService; - private final AggregationService aggregationService; - - public Mono> queryPager(QueryParam queryParam) { - return elasticSearchService.queryPager(ALARM_HISTORY_INDEX, queryParam, AlarmHistoryInfo.class); - } - - @Override - public Flux aggregation(AggregationQueryParam param) { - return aggregationService - .aggregation(ALARM_HISTORY_INDEX, param) - .map(AggregationData::of); - } - - public Mono save(AlarmHistoryInfo historyInfo) { - return elasticSearchService.commit(ALARM_HISTORY_INDEX, createData(historyInfo)); - } - - public Mono save(Flux historyInfo) { - return elasticSearchService.save(ALARM_HISTORY_INDEX, historyInfo.map(this::createData)); - } - - public Mono save(Mono historyInfo) { - return elasticSearchService.save(ALARM_HISTORY_INDEX, historyInfo.map(this::createData)); - } - - @Override - public Flux query(QueryParam param) { - return elasticSearchService - .query(ALARM_HISTORY_INDEX,param,AlarmHistoryInfo.class); - } - - @Override - public Mono count(QueryParam queryParam) { - return elasticSearchService.count(ALARM_HISTORY_INDEX,queryParam); - } - - private Map createData(AlarmHistoryInfo info) { - Map data = FastBeanCopier.copy(info, new HashMap<>(16), "termSpec"); - if (info.getTermSpec() != null) { - data.put("termSpec", JSONObject.toJSONString(info.getTermSpec())); - } - return data; - } - - public void init() { - indexManager.putIndex( - new DefaultElasticSearchIndexMetadata(ALARM_HISTORY_INDEX) - .addProperty("id", StringType.GLOBAL) - .addProperty("alarmConfigId", StringType.GLOBAL) - .addProperty("alarmConfigName", StringType.GLOBAL) - .addProperty("alarmRecordId", StringType.GLOBAL) - .addProperty("level", IntType.GLOBAL) - .addProperty("description", StringType.GLOBAL) - .addProperty("alarmTime", DateTimeType.GLOBAL) - .addProperty("targetType", StringType.GLOBAL) - .addProperty("targetName", StringType.GLOBAL) - .addProperty("targetId", StringType.GLOBAL) - - .addProperty("sourceType", StringType.GLOBAL) - .addProperty("sourceName", StringType.GLOBAL) - .addProperty("sourceId", StringType.GLOBAL) - - .addProperty("alarmInfo", StringType.GLOBAL) - .addProperty("creatorId", StringType.GLOBAL) - .addProperty("termSpec", StringType.GLOBAL) - .addProperty("triggerDesc", StringType.GLOBAL) - .addProperty("actualDesc", StringType.GLOBAL) - .addProperty("alarmConfigSource", StringType.GLOBAL) - ).block(Duration.ofSeconds(10)); - } -} diff --git a/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/service/LocalRuleInstanceRepository.java b/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/service/LocalRuleInstanceRepository.java index 0e523a74..b52640c3 100644 --- a/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/service/LocalRuleInstanceRepository.java +++ b/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/service/LocalRuleInstanceRepository.java @@ -78,14 +78,14 @@ public class LocalRuleInstanceRepository implements RuleInstanceRepository, Comm @Override public void run(String... args) throws Exception { - this - .findAll() - .flatMap(ruleInstance -> ruleEngine - .startRule(ruleInstance.getId(), ruleInstance.getModel()) - .onErrorResume(err -> { - log.warn("启动规则[{}]失败: {}", ruleInstance.getModel().getName(), ruleInstance); - return Mono.empty(); - })) - .subscribe(); +// this +// .findAll() +// .flatMap(ruleInstance -> ruleEngine +// .startRule(ruleInstance.getId(), ruleInstance.getModel()) +// .onErrorResume(err -> { +// log.warn("启动规则[{}]失败: {}", ruleInstance.getModel().getName(), ruleInstance); +// return Mono.empty(); +// })) +// .subscribe(); } } diff --git a/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/service/RuleInstanceService.java b/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/service/RuleInstanceService.java index 9af52754..fce71340 100644 --- a/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/service/RuleInstanceService.java +++ b/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/service/RuleInstanceService.java @@ -4,12 +4,11 @@ import lombok.extern.slf4j.Slf4j; import org.hswebframework.ezorm.core.param.QueryParam; import org.hswebframework.web.api.crud.entity.PagerResult; import org.hswebframework.web.crud.service.GenericReactiveCrudService; -import org.jetlinks.community.elastic.search.service.ElasticSearchService; import org.jetlinks.community.rule.engine.entity.RuleEngineExecuteEventInfo; import org.jetlinks.community.rule.engine.entity.RuleEngineExecuteLogInfo; import org.jetlinks.community.rule.engine.entity.RuleInstanceEntity; import org.jetlinks.community.rule.engine.enums.RuleInstanceState; -import org.jetlinks.community.rule.engine.event.handler.RuleEngineLoggerIndexProvider; +import org.jetlinks.community.rule.engine.log.RuleEngineLogService; import org.jetlinks.rule.engine.api.RuleEngine; import org.jetlinks.rule.engine.api.model.RuleEngineModelParser; import org.jetlinks.rule.engine.api.model.RuleModel; @@ -31,14 +30,14 @@ public class RuleInstanceService extends GenericReactiveCrudService> queryExecuteEvent(QueryParam queryParam) { - return elasticSearchService.queryPager(RuleEngineLoggerIndexProvider.RULE_EVENT_LOG, queryParam, RuleEngineExecuteEventInfo.class); + return ruleEngineLogService.queryEvent(queryParam); } public Mono> queryExecuteLog(QueryParam queryParam) { - return elasticSearchService.queryPager(RuleEngineLoggerIndexProvider.RULE_LOG, queryParam, RuleEngineExecuteLogInfo.class); + return ruleEngineLogService.queryLog(queryParam); } @Transactional diff --git a/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/service/TimeSeriesAlarmHistoryService.java b/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/service/TimeSeriesAlarmHistoryService.java new file mode 100644 index 00000000..84ce7361 --- /dev/null +++ b/jetlinks-manager/rule-engine-manager/src/main/java/org/jetlinks/community/rule/engine/service/TimeSeriesAlarmHistoryService.java @@ -0,0 +1,129 @@ +package org.jetlinks.community.rule.engine.service; + +import lombok.AllArgsConstructor; +import org.hswebframework.ezorm.core.param.QueryParam; +import org.hswebframework.web.api.crud.entity.PagerResult; +import org.hswebframework.web.bean.FastBeanCopier; +import org.jetlinks.core.metadata.SimplePropertyMetadata; +import org.jetlinks.core.metadata.types.ArrayType; +import org.jetlinks.core.metadata.types.DateTimeType; +import org.jetlinks.core.metadata.types.IntType; +import org.jetlinks.core.metadata.types.StringType; +import org.jetlinks.community.ConfigMetadataConstants; +import org.jetlinks.community.rule.engine.entity.AlarmHistoryInfo; +import org.jetlinks.community.timeseries.TimeSeriesData; +import org.jetlinks.community.timeseries.TimeSeriesManager; +import org.jetlinks.community.timeseries.TimeSeriesMetadata; +import org.jetlinks.community.timeseries.TimeSeriesMetric; +import org.jetlinks.community.timeseries.query.AggregationData; +import org.jetlinks.community.timeseries.query.AggregationQueryParam; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import javax.annotation.PostConstruct; +import java.time.Duration; +import java.util.HashMap; +import java.util.Map; + +/** + * @author zhouhao + * @since 2.2 + */ +@AllArgsConstructor +public class TimeSeriesAlarmHistoryService implements AlarmHistoryService { + + //兼容es索引 + public final static String ALARM_HISTORY_METRIC = "alarm_history"; + + private final TimeSeriesManager manager; + + @Override + public Flux aggregation(AggregationQueryParam param) { + return manager + .getService(ALARM_HISTORY_METRIC) + .aggregation(param); + } + + @Override + public Flux query(QueryParam param) { + return manager + .getService(ALARM_HISTORY_METRIC) + .query(param) + .map(ts -> ts.as(AlarmHistoryInfo.class)); + } + + @Override + public Mono count(QueryParam queryParam) { + return manager + .getService(ALARM_HISTORY_METRIC) + .count(queryParam) + .map(Number::longValue); + } + + public Mono> queryPager(QueryParam queryParam) { + return manager + .getService(ALARM_HISTORY_METRIC) + .queryPager(queryParam, ts -> ts.as(AlarmHistoryInfo.class)); + } + + public Mono save(AlarmHistoryInfo historyInfo) { + return manager + .getService(ALARM_HISTORY_METRIC) + .commit(createData(historyInfo)); + } + + public Mono save(Flux historyInfo) { + return manager + .getService(ALARM_HISTORY_METRIC) + .save(historyInfo.map(this::createData)); + } + + public Mono save(Mono historyInfo) { + return save(historyInfo.flux()); + } + + private TimeSeriesData createData(AlarmHistoryInfo info) { + Map data = FastBeanCopier.copy(info, new HashMap<>(16)); + + return TimeSeriesData.of(info.getAlarmTime(), data); + + } + + @PostConstruct + public void init() { + // 大字符,在某些关系型数据库的实现上,使用longvarchar存储. + StringType longStringType = new StringType() + .expand(ConfigMetadataConstants.maxLength, 8000L); + + manager.registerMetadata( + TimeSeriesMetadata.of( + TimeSeriesMetric.of(ALARM_HISTORY_METRIC), + SimplePropertyMetadata.of("id", "ID", StringType.GLOBAL), + + SimplePropertyMetadata.of("alarmConfigId", "ID", StringType.GLOBAL), + SimplePropertyMetadata.of("alarmConfigName", "ID", StringType.GLOBAL), + SimplePropertyMetadata.of("alarmRecordId", "ID", StringType.GLOBAL), + + SimplePropertyMetadata.of("level", "level", IntType.GLOBAL), + SimplePropertyMetadata.of("description", "description", StringType.GLOBAL), + SimplePropertyMetadata.of("alarmTime", "alarmTime", DateTimeType.GLOBAL), + + SimplePropertyMetadata.of("targetType", "targetType", StringType.GLOBAL), + SimplePropertyMetadata.of("targetName", "targetName", StringType.GLOBAL), + SimplePropertyMetadata.of("targetId", "targetId", StringType.GLOBAL), + + SimplePropertyMetadata.of("sourceType", "sourceType", StringType.GLOBAL), + SimplePropertyMetadata.of("sourceName", "sourceName", StringType.GLOBAL), + SimplePropertyMetadata.of("sourceId", "sourceId", StringType.GLOBAL), + + SimplePropertyMetadata.of("alarmInfo", "alarmInfo", longStringType), + SimplePropertyMetadata.of("creatorId", "creatorId", StringType.GLOBAL), + SimplePropertyMetadata.of("termSpec", "termSpec", longStringType), + SimplePropertyMetadata.of("triggerDesc", "triggerDesc", longStringType), + SimplePropertyMetadata.of("actualDesc", "actualDesc", longStringType), + SimplePropertyMetadata.of("alarmConfigSource", "alarmConfigSource", StringType.GLOBAL), + SimplePropertyMetadata.of("bindings", "bindings", new ArrayType().elementType(StringType.GLOBAL)) + + )).block(Duration.ofSeconds(10)); + } +} diff --git a/jetlinks-manager/rule-engine-manager/src/main/resources/i18n/rule-engine-manager/messages_zh.properties b/jetlinks-manager/rule-engine-manager/src/main/resources/i18n/rule-engine-manager/messages_zh.properties index 5520e465..1606fa73 100644 --- a/jetlinks-manager/rule-engine-manager/src/main/resources/i18n/rule-engine-manager/messages_zh.properties +++ b/jetlinks-manager/rule-engine-manager/src/main/resources/i18n/rule-engine-manager/messages_zh.properties @@ -71,8 +71,6 @@ org.jetlinks.community.rule.engine.enums.AlarmState.disabled=\u7981\u7528 org.jetlinks.community.rule.engine.enums.RuleInstanceState.started=\u6B63\u5E38 org.jetlinks.community.rule.engine.enums.RuleInstanceState.disable=\u7981\u7528 -org.jetlinks.community.rule.engine.enums.SqlRuleType.timer=\u5B9A\u65F6 -org.jetlinks.community.rule.engine.enums.SqlRuleType.realTime=\u5B9E\u65F6 org.jetlinks.community.rule.engine.device.DeviceAlarmRule.TriggerType.device=\u8BBE\u5907\u6D88\u606F org.jetlinks.community.rule.engine.device.DeviceAlarmRule.TriggerType.timer=\u5B9A\u65F6 diff --git a/jetlinks-standalone/Dockerfile b/jetlinks-standalone/Dockerfile index fa293e22..1cdc47a4 100644 --- a/jetlinks-standalone/Dockerfile +++ b/jetlinks-standalone/Dockerfile @@ -1,21 +1,15 @@ -FROM openjdk:8u272-jdk as builder +FROM jetlinks/base-jdk-21:latest as builder WORKDIR application -ARG JAR_FILE=target/jetlinks-standalone.jar -COPY ${JAR_FILE} application.jar -RUN java -Djarmode=layertools -jar application.jar extract +ARG JAR_FILE=target/application.jar +COPY ${JAR_FILE} app.jar +RUN java -Djarmode=tools -jar app.jar extract --layers --launcher +FROM jetlinks/base-jdk-21:latest -FROM openjdk:8u272-jdk WORKDIR application -COPY --from=builder application/dependencies/ ./ -RUN true -COPY --from=builder application/snapshot-dependencies/ ./ -RUN true -COPY --from=builder application/spring-boot-loader/ ./ -RUN true -COPY --from=builder application/application/ ./ -RUN true +COPY --from=builder application/app/dependencies/ ./ +COPY --from=builder application/app/snapshot-dependencies/ ./ +COPY --from=builder application/app/spring-boot-loader/ ./ +COPY --from=builder application/app/application/ ./ COPY docker-entrypoint.sh ./ -RUN true RUN chmod +x docker-entrypoint.sh -RUN true ENTRYPOINT ["./docker-entrypoint.sh"] \ No newline at end of file diff --git a/jetlinks-standalone/docker-entrypoint.sh b/jetlinks-standalone/docker-entrypoint.sh index ddb199da..c06a0e8f 100755 --- a/jetlinks-standalone/docker-entrypoint.sh +++ b/jetlinks-standalone/docker-entrypoint.sh @@ -1,7 +1,21 @@ #!/usr/bin/env bash java $JAVA_OPTS -server \ --XX:+UnlockExperimentalVMOptions \ --XX:+UseCGroupMemoryLimitForHeap \ --XX:-OmitStackTraceInFastThrow \ +--add-opens \ +java.base/java.lang=ALL-UNNAMED \ +--add-opens \ +java.base/java.util=ALL-UNNAMED \ +--add-opens \ +java.base/java.util.concurrent=ALL-UNNAMED \ +--add-opens \ +java.base/java.io=ALL-UNNAMED \ +--add-opens \ +java.base/java.net=ALL-UNNAMED \ +--add-opens \ +java.base/java.text=ALL-UNNAMED \ +--add-opens \ +java.base/java.math=ALL-UNNAMED \ +--add-opens \ +java.scripting/javax.script=ALL-UNNAMED \ +-Dreactor.schedulers.defaultBoundedElasticOnVirtualThreads=true \ -Djava.security.egd=file:/dev/./urandom \ -org.springframework.boot.loader.JarLauncher \ No newline at end of file +org.springframework.boot.loader.launch.PropertiesLauncher \ No newline at end of file diff --git a/jetlinks-standalone/pom.xml b/jetlinks-standalone/pom.xml index d0ae684e..b63ea9a2 100644 --- a/jetlinks-standalone/pom.xml +++ b/jetlinks-standalone/pom.xml @@ -5,7 +5,7 @@ jetlinks-community org.jetlinks.community - 2.3.0-SNAPSHOT + 2.10.0-SNAPSHOT ../pom.xml 4.0.0 @@ -53,25 +53,59 @@ + - java11 + es7x - 11 - 11 - 11 - 11 + 7.17.26 + - org.openjdk.nashorn - nashorn-core - 15.4 + org.jetlinks.community + elasticsearch-7x + + + + co.elastic.clients + elasticsearch-java + ${elasticsearch.version} + + + + org.elasticsearch.client + elasticsearch-rest-client + ${elasticsearch.version} + + + + + + + es8x + + + org.jetlinks.community + elasticsearch-8x + + + org.jetlinks.community + common-component + + + + + + org.jetlinks.community + timescaledb-component + + com.fasterxml.jackson.dataformat @@ -189,7 +223,6 @@ io.asyncer r2dbc-mysql - 0.9.3 @@ -218,11 +251,6 @@ spring-boot-starter-webflux - - org.elasticsearch.client - elasticsearch-rest-high-level-client - - org.springframework.boot spring-boot-starter diff --git a/jetlinks-standalone/src/main/java/org/jetlinks/community/standalone/JetLinksApplication.java b/jetlinks-standalone/src/main/java/org/jetlinks/community/standalone/JetLinksApplication.java index bb54fbb0..940a5389 100644 --- a/jetlinks-standalone/src/main/java/org/jetlinks/community/standalone/JetLinksApplication.java +++ b/jetlinks-standalone/src/main/java/org/jetlinks/community/standalone/JetLinksApplication.java @@ -2,20 +2,13 @@ package org.jetlinks.community.standalone; import lombok.extern.slf4j.Slf4j; import org.hswebframework.web.authorization.basic.configuration.EnableAopAuthorize; -import org.hswebframework.web.authorization.events.AuthorizingHandleBeforeEvent; import org.hswebframework.web.crud.annotation.EnableEasyormRepository; import org.hswebframework.web.logging.aop.EnableAccessLogger; -import org.hswebframework.web.logging.events.AccessLoggerAfterEvent; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.elasticsearch.ElasticsearchRestClientAutoConfiguration; import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; import org.springframework.cache.annotation.EnableCaching; -import org.springframework.context.annotation.Profile; -import org.springframework.context.event.EventListener; -import org.springframework.stereotype.Component; - -import javax.annotation.PostConstruct; @SpringBootApplication(scanBasePackages = "org.jetlinks.community", exclude = { @@ -33,25 +26,6 @@ public class JetLinksApplication { SpringApplication.run(JetLinksApplication.class, args); } - @Component - @Slf4j - public static class AdminAllAccess { - - @EventListener - public void handleAuthEvent(AuthorizingHandleBeforeEvent e) { - //admin用户拥有所有权限 - if (e.getContext().getAuthentication().getUser().getUsername().equals("admin")) { - e.setAllow(true); - } - } - - @EventListener - public void handleAccessLogger(AccessLoggerAfterEvent event) { - - log.info("{}=>{} {}-{}", event.getLogger().getIp(), event.getLogger().getUrl(), event.getLogger().getDescribe(), event.getLogger().getAction()); - - } - } } diff --git a/jetlinks-standalone/src/main/resources/application-embedded.yml b/jetlinks-standalone/src/main/resources/application-embedded.yml deleted file mode 100644 index 8585e8ff..00000000 --- a/jetlinks-standalone/src/main/resources/application-embedded.yml +++ /dev/null @@ -1,44 +0,0 @@ -spring: - resources: - static-locations: file:./index/, file:./static/,/,classpath:/META-INF/resources/,classpath:/resources/,classpath:/static/, classpath:/public/ - redis: - host: 127.0.0.1 - port: 6379 - lettuce: - pool: - max-active: 1024 - timeout: 20s - r2dbc: - url: r2dbc:h2:file:///./data/h2db/jetlinks - username: sa - password: - pool: - max-size: 32 - elasticsearch: - uris: localhost:9201 - socket-timeout: 10s - connection-timeout: 15s - webclient: - max-in-memory-size: 100MB -easyorm: - default-schema: PUBLIC # 数据库默认的schema - dialect: h2 #数据库方言 -elasticsearch: - embedded: - enabled: true # 为true时使用内嵌的elasticsearch,不建议在生产环境中使用 - data-path: ./data/elasticsearch - port: 9201 - host: 0.0.0.0 - index: - default-strategy: time-by-month #默认es的索引按月进行分表, direct则为直接操作索引. - settings: - number-of-shards: 1 # es 分片数量 - number-of-replicas: 0 # 副本数量 -device: - message: - writer: - time-series: - enabled: true #写出设备消息数据到elasticsearch -logging: - level: - com.github.tonivade: error \ No newline at end of file diff --git a/jetlinks-standalone/src/main/resources/application.yml b/jetlinks-standalone/src/main/resources/application.yml index d100ca2f..64d6d2b2 100644 --- a/jetlinks-standalone/src/main/resources/application.yml +++ b/jetlinks-standalone/src/main/resources/application.yml @@ -1,236 +1,267 @@ server: - port: 8848 + port: 8848 spring: - profiles: - active: dev - application: - name: jetlinks-platform - jackson: - date-format: yyyy-MM-dd HH:mm:ss - time-zone: Asia/Shanghai - serialization: - WRITE_DATES_AS_TIMESTAMPS: true - default-property-inclusion: non_null - codec: - max-in-memory-size: 100MB - web: - resources: - static-locations: file:./static/,/,classpath:/META-INF/resources/,classpath:/resources/,classpath:/static/, classpath:/public/ - redis: - host: 127.0.0.1 - port: 6379 - lettuce: - pool: - max-active: 1024 - timeout: 20s - # database: 3 - # max-wait: 10s - r2dbc: - # 需要手动创建数据库,启动会自动创建表,修改了配置easyorm相关配置也要修改 - url: r2dbc:postgresql://127.0.0.1:5432/jetlinks - # url: r2dbc:mysql://127.0.0.1:3306/jetlinks?ssl=false&serverZoneId=Asia/Shanghai # 修改了配置easyorm相关配置也要修改 - username: postgres - password: jetlinks - pool: - max-size: 32 - max-idle-time: 2m # 值不能大于mysql server的wait_timeout配置 - max-life-time: 10m - acquire-retry: 3 - reactor: - debug-agent: - enabled: false - elasticsearch: - uris: 127.0.0.1:9200 - socket-timeout: 10s - connection-timeout: 15s - webclient: - max-in-memory-size: 100MB + profiles: + active: dev + application: + name: jetlinks-platform + jackson: + date-format: yyyy-MM-dd HH:mm:ss + time-zone: Asia/Shanghai + serialization: + WRITE_DATES_AS_TIMESTAMPS: true + default-property-inclusion: non_null + codec: + max-in-memory-size: 100MB + web: + resources: + static-locations: file:./static/,/,classpath:/META-INF/resources/,classpath:/resources/,classpath:/static/, classpath:/public/ + data: + redis: + host: 127.0.0.1 + port: 6379 + lettuce: + pool: + max-active: 1024 + timeout: 20s + # database: 3 + # max-wait: 10s + r2dbc: + # 需要手动创建数据库,启动会自动创建表,修改了配置easyorm相关配置也要修改 + url: r2dbc:postgresql://127.0.0.1:5432/jetlinks + # url: r2dbc:mysql://127.0.0.1:3306/jetlinks?ssl=false&serverZoneId=Asia/Shanghai # 修改了配置easyorm相关配置也要修改 + username: postgres + password: jetlinks + pool: + max-size: 128 + max-idle-time: 2m # 值不能大于mysql server的wait_timeout配置 + max-life-time: 10m + max-acquire-time: 30s + reactor: + debug-agent: + enabled: false +# elasticsearch: +# uris: 127.0.0.1:9200 +# socket-timeout: 10s +# connection-timeout: 15s easyorm: - default-schema: public # 数据库默认的schema - dialect: postgres #数据库方言 + default-schema: public # 数据库默认的schema + dialect: postgres #数据库方言 +timescaledb: + enabled: true + shared-spring: true # 默认共享spring的连接 + # r2dbc: + # url: r2dbc:postgresql://localhost:15432/jetlinks + # username: postgres + # password: p@ssw0rd + schema: ${easyorm.default-schema} + time-series: + enabled: true + retention-policies: + # 设备会话统计，只保留30天 + - table: device_session_metric + interval: 30d + # 设备消息统计，只保留30天 + - table: device_metrics + interval: 30d + things-data: + enabled: true # 开启设备数据存储 +# retention-policy: 30d # 保留策略,30天 +# chunk-time-interval: 1d # 分区时间间隔,1天 tdengine: - enabled: false - database: jetlinks - restful: - endpoints: - - http://127.0.0.1:6041/ - username: root - password: taosdata + enabled: false # 使用tdengine来存储设备数据 + database: jetlinks + restful: + endpoints: + - http://127.0.0.1:6041/ + username: root + password: taosdata elasticsearch: - embedded: - enabled: false # 为true时使用内嵌的elasticsearch,不建议在生产环境中使用 - data-path: ./data/elasticsearch - port: 9200 - host: 0.0.0.0 - index: - default-strategy: time-by-month #默认es的索引按月进行分表, direct则为直接操作索引. - settings: - number-of-shards: 1 # es 分片数量 - number-of-replicas: 0 # 副本数量 + index: + default-strategy: time-by-month #默认es的索引按月进行分表, direct则为直接操作索引. + settings: + number-of-shards: 1 # es 分片数量 + number-of-replicas: 0 # 副本数量 device: - message: - writer: - time-series: - enabled: true #对设备数据进行持久化 + message: + writer: + time-series: + enabled: true #对设备数据进行持久化 + captcha: - enabled: false # 开启验证码 - ttl: 2m #验证码过期时间,2分钟 + enabled: false # 开启验证码 + ttl: 2m #验证码过期时间,2分钟 hsweb: - cors: - enable: true - configs: - - path: /** - allowed-headers: "*" - allowed-methods: [ "GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS" ] - allowed-origins: [ "*" ] ## 生产环境请替换为具体的域名端口如: http://xxxxx - max-age: 1800 - dict: - enum-packages: org.jetlinks - file: - upload: - static-file-path: ./static/upload - static-location: http://127.0.0.1:${server.port}/upload - webflux: - response-wrapper: - enabled: true #开启响应包装器(将返回值包装为ResponseMessage) - excludes: # 这下包下的接口不包装 - - org.springdoc - # auth: #默认的用户配置 - # users: - # admin: - # username: admin - # password: admin - # name: 超级管理员 - authorize: - auto-parse: true -# user-token: -# allopatric-login-mode: offlineOther # 设置异地登录模式为 将其他地方登录的相同用户踢下线 -# allopatric-login-modes: -# app: offlineOther - permission: - filter: - enabled: true # 设置为true开启权限过滤,赋权时,不能赋予比自己多的权限. - exclude-username: admin # admin用户不受上述限制 - un-auth-strategy: ignore # error表示:发生越权时,抛出403错误. ignore表示会忽略越权的赋权. - cache: - type: redis - redis: - local-cache-type: guava + cors: + enable: true + configs: + - path: /** + allowed-headers: "*" + allowed-methods: [ "GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS" ] + allowed-origins: [ "*" ] ## 生产环境请替换为具体的域名端口如: http://xxxxx + max-age: 1800 + dict: + enum-packages: org.jetlinks + webflux: + response-wrapper: + enabled: true #开启响应包装器(将返回值包装为ResponseMessage) + excludes: # 这下包下的接口不包装 + - org.springdoc + authorize: + auto-parse: true + # user-token: + # allopatric-login-mode: offlineOther # 设置异地登录模式为 将其他地方登录的相同用户踢下线 + # allopatric-login-modes: + # app: offlineOther + permission: + filter: + enabled: true # 设置为true开启权限过滤,赋权时,不能赋予比自己多的权限. + exclude-username: admin # admin用户不受上述限制 + un-auth-strategy: ignore # error表示:发生越权时,抛出403错误. ignore表示会忽略越权的赋权. + cache: + type: redis + redis: + local-cache-type: guava file: - manager: - storage-base-path: ./data/files + manager: + storage-base-path: ./data/files api: - # 访问api接口的根地址 - base-path: http://127.0.0.1:${server.port} + # 访问api接口的根地址 + base-path: http://127.0.0.1:${server.port} jetlinks: - server-id: ${spring.application.name}:${server.port} #设备服务网关服务ID,不同服务请设置不同的ID - logging: - system: - context: - server: ${spring.application.name} - protocol: - spi: - enabled: true # 为true时开启自动加载通过依赖引入的协议包 + server-id: ${spring.application.name}:${server.port} #设备服务网关服务ID,不同服务请设置不同的ID + logging: + system: + context: + server: ${spring.application.name} + device: + storage: + default-policy: timescaledb-row # 默认设备数据存储策略 + enable-last-data-in-db: false # 是否将设备最新到数据存储到数据库 + authentication: + defaults: + user: + admin: + - "*:*" + user-init: + enabled: true + users: # 系统初始用户密码 + - username: admin + password: ${ADMIN_USER_PASSWORD:JetLinks.C0mmVn1ty} + name: Administrator + type: admin +rule: + engine: + server-id: ${jetlinks.server-id} + server-name: ${spring.application.name} logging: - level: - org.jetlinks: debug - rule.engine: debug - org.hswebframework: debug - org.springframework.transaction: debug - org.springframework.data.r2dbc.connectionfactory: warn - io.micrometer: warn - org.hswebframework.expands: error - system: debug - org.jetlinks.rule.engine: warn - org.jetlinks.supports.event: warn - org.springframework: warn - org.jetlinks.community.device.message.writer: warn - org.jetlinks.community.timeseries.micrometer: warn - org.jetlinks.community.elastic.search.service.reactive: trace - org.jetlinks.community.network: warn - io.vertx.mqtt.impl: warn - org.jetlinks.supports.scalecube.rpc: warn - "org.jetlinks.community.buffer": debug - org.elasticsearch: error - org.elasticsearch.deprecation: off - "io.vertx.core.impl.ContextImpl": off - "org.hswebframework.web.starter.jackson": warn - config: classpath:logback-spring.xml + level: + org.jetlinks: debug + rule.engine: debug + org.hswebframework: debug + org.springframework.transaction: debug + org.springframework.data.r2dbc.connectionfactory: warn + io.micrometer: warn + org.hswebframework.expands: error + system: debug + org.jetlinks.rule.engine: warn + org.jetlinks.supports.event: warn + org.springframework: warn + org.jetlinks.community.device.message.writer: warn + org.jetlinks.community.timeseries.micrometer: warn + org.jetlinks.community.elastic.search.service.reactive: trace + org.jetlinks.community.network: warn + io.vertx.mqtt.impl: warn + org.jetlinks.supports.scalecube.rpc: warn + "org.jetlinks.community.buffer": debug + org.elasticsearch: error + org.elasticsearch.deprecation: off + "io.vertx.core.impl.ContextImpl": off + "org.hswebframework.web.starter.jackson": warn + config: classpath:logback-spring.xml vertx: - max-event-loop-execute-time-unit: seconds - max-event-loop-execute-time: 30 - max-worker-execute-time-unit: seconds - max-worker-execute-time: 30 - prefer-native-transport: true + max-event-loop-execute-time-unit: seconds + max-event-loop-execute-time: 30 + max-worker-execute-time-unit: seconds + max-worker-execute-time: 30 + prefer-native-transport: true micrometer: - time-series: - tags: - server: ${spring.application.name} - metrics: - default: - step: 30s + time-series: + tags: + server: ${spring.application.name} + metrics: + default: + step: 30s system: - config: - scopes: - - id: front - name: 前端配置 - public-access: true - - id: paths - name: 访问路径配置 - public-access: true - properties: - - key: base-path - name: 接口根路径 - default-value: http://127.0.0.1:9000/api - - id: amap - name: 高德地图配置 - public-access: false - properties: - - key: apiKey # 配置id - name: 高德地图ApiKey # 名称 + config: + scopes: + - id: front + name: 前端配置 + public-access: true + - id: paths + name: 访问路径配置 + public-access: true + properties: + - key: base-path + name: 接口根路径 + default-value: http://127.0.0.1:9000/api + - id: amap + name: 高德地图配置 + public-access: false + properties: + - key: apiKey # 配置id + name: 高德地图ApiKey # 名称 management: - health: - elasticsearch: - enabled: false # 关闭elasticsearch健康检查 + health: + elasticsearch: + enabled: false + elastic: + metrics: + export: + enabled: false + simple: + metrics: + export: + enabled: false springdoc: - swagger-ui: - path: /swagger-ui.html - # packages-to-scan: org.jetlinks - group-configs: - - group: 设备管理相关接口 - packages-to-scan: - - org.jetlinks.community.device - paths-to-exclude: - - /device-instance/** - - /device-product/** - - /protocol/** - - group: 规则引擎相关接口 - packages-to-scan: org.jetlinks.community.rule.engine.web - paths-to-exclude: /api/** - - group: 通知管理相关接口 - packages-to-scan: org.jetlinks.community.notify.manager.web - - group: 设备接入相关接口 - packages-to-scan: - - org.jetlinks.community.network.manager.web - - org.jetlinks.community.device.web - paths-to-match: - - /gateway/** - - /network/** - - /protocol/** - - group: 系统管理相关接口 - packages-to-scan: - - org.jetlinks.community.auth - - org.hswebframework.web.system.authorization.defaults.webflux - - org.hswebframework.web.file - - org.jetlinks.community.io.file.web - - org.hswebframework.web.authorization.basic.web - - org.jetlinks.community.logging.controller - cache: - disabled: false + swagger-ui: + path: /swagger-ui.html + # packages-to-scan: org.jetlinks + group-configs: + - group: 设备管理相关接口 + packages-to-scan: + - org.jetlinks.community.device + paths-to-exclude: + - /device-instance/** + - /device-product/** + - /protocol/** + - group: 规则引擎相关接口 + packages-to-scan: org.jetlinks.community.rule.engine.web + paths-to-exclude: /api/** + - group: 通知管理相关接口 + packages-to-scan: org.jetlinks.community.notify.manager.web + - group: 设备接入相关接口 + packages-to-scan: + - org.jetlinks.community.network.manager.web + - org.jetlinks.community.device.web + paths-to-match: + - /gateway/** + - /network/** + - /protocol/** + - group: 系统管理相关接口 + packages-to-scan: + - org.jetlinks.community.auth + - org.hswebframework.web.system.authorization.defaults.webflux + - org.hswebframework.web.file + - org.jetlinks.community.io.file.web + - org.hswebframework.web.authorization.basic.web + - org.jetlinks.community.logging.controller + cache: + disabled: false network: - resources: - - 1883-1890 - - 8800-8810 - - 5060-5061 + resources: + - 1883-1890 + - 8800-8810 + - 5060-5061 diff --git a/jetlinks-standalone/src/main/resources/banner.txt b/jetlinks-standalone/src/main/resources/banner.txt index 6235ab7b..630ae293 100644 --- a/jetlinks-standalone/src/main/resources/banner.txt +++ b/jetlinks-standalone/src/main/resources/banner.txt @@ -6,6 +6,4 @@ \____/ \___|\__|______|_|_| |_|_|\_\___/ jetlinks @project.version@ build @maven.build.timestamp@ spring-boot ${spring-boot.version} - http port ${server.port} - redis ${spring.redis.host}:${spring.redis.port} - r2dbc ${spring.r2dbc.url} \ No newline at end of file + http port ${server.port} \ No newline at end of file diff --git a/jetlinks-standalone/src/main/resources/hsweb-starter.js b/jetlinks-standalone/src/main/resources/hsweb-starter.js deleted file mode 100644 index 20fdba42..00000000 --- a/jetlinks-standalone/src/main/resources/hsweb-starter.js +++ /dev/null @@ -1,53 +0,0 @@ -//组件信息 -var info = { - groupId: "org.jetlinks", - artifactId: "jetlinks-community", - version: "2.0.0", - website: "http://github.com/jetlinks/jetlinks-community", - comment: "jetlinks" -}; - -var users = [{ - "id" : "1199596756811550720", - "username" : "admin", - "password": "104ffe90cd840e08f7a79c7fddbe1699", - "salt": "LmKOhcoB", - "status":1, - "name": "超级管理员", - "type":"admin" -}]; -//版本更新信息 -var versions = [ - { - version: "1.0.1", - upgrade: function (context) { - - } - } -]; - -function initialize(context) { - var database = context.database; - - database.dml().upsert("s_user").values(users).execute().sync(); -} - -function install(context) { - - -} - - -//设置依赖 -dependency.setup(info) - .onInstall(install) - .onUpgrade(function (context) { //更新时执行 - var upgrader = context.upgrader; - upgrader.filter(versions) - .upgrade(function (newVer) { - newVer.upgrade(context); - }); - }) - .onUninstall(function (context) { //卸载时执行 - - }).onInitialize(initialize); diff --git a/jetlinks-standalone/src/main/resources/logback-spring.xml b/jetlinks-standalone/src/main/resources/logback-spring.xml index a5950057..aedcff2f 100644 --- a/jetlinks-standalone/src/main/resources/logback-spring.xml +++ b/jetlinks-standalone/src/main/resources/logback-spring.xml @@ -1,5 +1,10 @@ + + + + diff --git a/pom.xml b/pom.xml index dbd2fd05..b7e1661f 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.jetlinks.community jetlinks-community - 2.3.0-SNAPSHOT + 2.10.0-SNAPSHOT jetlinks-components jetlinks-manager @@ -175,13 +175,6 @@ - - - org.codehaus.groovy - groovy - 2.5.14 - - @@ -205,6 +198,12 @@ + + org.jetlinks.community + common-component + ${project.version} + + org.springframework spring-framework-bom @@ -229,12 +228,6 @@ pom - - io.asyncer - r2dbc-mysql - 0.9.3 - - de.ruedigermoeller fst @@ -398,6 +391,12 @@ org.jetlinks reactor-ql ${reactor.ql.version} + + + commons-logging + commons-logging + + @@ -441,10 +440,68 @@ io.opentelemetry - opentelemetry-semconv - ${opentelemetry.version}-alpha + opentelemetry-exporter-logging + ${opentelemetry.version} + + io.opentelemetry + opentelemetry-sdk-trace + ${opentelemetry.version} + + + + io.opentelemetry + opentelemetry-sdk + ${opentelemetry.version} + + + + io.opentelemetry.instrumentation + opentelemetry-instrumentation-bom-alpha + 2.5.0-alpha + pom + import + + + + io.opentelemetry + opentelemetry-semconv + 1.29.0-alpha + + + + io.opentelemetry + opentelemetry-exporter-jaeger + 1.34.1 + + + + commons-beanutils + commons-beanutils + 1.10.1 + + + commons-logging + commons-logging + + + + + + org.apache.commons + commons-configuration2 + 2.11.0 + + + commons-logging + commons-logging + + + + + + io.r2dbc r2dbc-bom @@ -509,6 +566,24 @@ 32.1.2-jre + + org.jetlinks.community + elasticsearch-core + ${project.version} + + + + org.jetlinks.community + elasticsearch-8x + ${project.version} + + + + org.jetlinks.community + elasticsearch-7x + ${project.version} + + org.elasticsearch.client elasticsearch-rest-high-level-client @@ -522,15 +597,33 @@ - org.elasticsearch.client - elasticsearch-rest-client - ${elasticsearch.version} + org.jetlinks.community + things-component + ${project.version} - org.elasticsearch.plugin - transport-netty4-client + org.jetlinks.community + timeseries-component + ${project.version} + + + + org.jetlinks.community + timescaledb-component + ${project.version} + + + + org.elasticsearch.client + elasticsearch-rest-client ${elasticsearch.version} + + + commons-logging + commons-logging + + @@ -567,6 +660,29 @@ com.aliyun aliyun-java-sdk-core ${aliyun.sdk.core} + + + commons-logging + commons-logging + + + + + + com.cronutils + cron-utils + 9.2.1 + compile + + + org.glassfish + jakarta.el + + + org.glassfish + javax.el + + @@ -575,6 +691,41 @@ ${grpc.version} + + io.asyncer + r2dbc-mysql + 1.4.0 + + + + org.postgresql + r2dbc-postgresql + 1.0.7.RELEASE + + + + io.r2dbc + r2dbc-h2 + 1.0.0.RELEASE + + + + io.r2dbc + r2dbc-spi + 1.0.0.RELEASE + + + + io.r2dbc + r2dbc-pool + 1.0.2.RELEASE + + + + org.glassfish.expressly + expressly + 6.0.0-M1 + io.grpc @@ -665,11 +816,6 @@ reactor-core - - org.codehaus.groovy - groovy - - junit junit @@ -726,18 +872,6 @@ - - - releases - Nexus Release Repository - https://nexus.jetlinks.cn/content/repositories/releases/ - - - snapshots - Nexus Snapshot Repository - https://nexus.jetlinks.cn/content/repositories/snapshots/ - -