suguo
/
jetlinks-community
mirror of https://gitee.com/jetlinks/jetlinks-community.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master'

This commit is contained in:
zhou-hao 2021-05-28 17:43:41 +08:00
parent 1919e563d8 b409932ac9
commit c93df4715b
25 changed files with 913 additions and 579 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
.github/workflows/pull_request.yml vendored Normal file
View File

@ -0,0 +1,27 @@
# This workflow will build a Java project with Maven
# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven
name: Java CI with Maven
on:
pull_request:
branches: [ master ]
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v1
- name: Set up JDK 1.8
uses: actions/setup-java@v1
with:
java-version: 1.8
- name: Cache Maven Repository
uses: actions/cache@v1
with:
path: ~/.m2
key: jetlinks-community-maven-repository
- name: Build with Maven
run: ./mvnw package -Dmaven.test.skip=true -Pbuild

1
jetlinks-components/dashboard-component/src/main/java/org/jetlinks/community/dashboard/MeasurementParameter.java
View File

@ -17,6 +17,7 @@ import java.util.Optional;
public class MeasurementParameter implements ValueObject {
private Map<String, Object> params = new HashMap<>();
@Override
public Optional<Object> get(String name) {
return Optional.ofNullable(params).map(p -> p.get(name));
}

5
jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/DefaultElasticSearchService.java
View File

@ -37,7 +37,6 @@ import org.jetlinks.community.elastic.search.utils.ReactorActionListener;
import org.jetlinks.core.utils.FluxUtils;
import org.reactivestreams.Publisher;
import org.springframework.context.annotation.DependsOn;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import reactor.core.publisher.BufferOverflowStrategy;
import reactor.core.publisher.Flux;
@ -119,12 +118,14 @@ public class DefaultElasticSearchService implements ElasticSearchService {
});
}
@Override
public <T> Flux<T> query(String index, QueryParam queryParam, Function<Map<String, Object>, T> mapper) {
return this
.doQuery(new String[]{index}, queryParam)
.flatMapMany(tp2 -> convertQueryResult(tp2.getT1(), tp2.getT2(), mapper));
}
@Override
public <T> Flux<T> query(String[] index, QueryParam queryParam, Function<Map<String, Object>, T> mapper) {
return this
.doQuery(index, queryParam)
@ -373,7 +374,7 @@ public class DefaultElasticSearchService implements ElasticSearchService {
private Mono<SearchResponse> doSearch(SearchRequest request) {
return this
.<SearchRequest, SearchResponse>execute(request, restClient.getQueryClient()::searchAsync)
.execute(request, restClient.getQueryClient()::searchAsync)
.onErrorResume(err -> {
log.error("query elastic error", err);
return Mono.empty();

5
jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/ElasticSearchService.java
View File

@ -12,6 +12,11 @@ import java.util.Collection;
import java.util.Map;
import java.util.function.Function;
/**
* ES数据库业务操作类
*
* @author zhouhao
*/
public interface ElasticSearchService {
default <T> Mono<PagerResult<T>> queryPager(String index, QueryParam queryParam, Function<Map<String, Object>, T> mapper) {

158
jetlinks-components/elasticsearch-component/src/main/java/org/jetlinks/community/elastic/search/service/reactive/ReactiveElasticSearchService.java
View File

@ -27,12 +27,12 @@ import org.hswebframework.utils.time.DateFormatter;
import org.hswebframework.utils.time.DefaultDateFormatter;
import org.hswebframework.web.api.crud.entity.PagerResult;
import org.hswebframework.web.bean.FastBeanCopier;
import org.jetlinks.core.utils.FluxUtils;
import org.jetlinks.community.elastic.search.index.ElasticSearchIndexManager;
import org.jetlinks.community.elastic.search.index.ElasticSearchIndexMetadata;
import org.jetlinks.community.elastic.search.service.ElasticSearchService;
import org.jetlinks.community.elastic.search.utils.ElasticSearchConverter;
import org.jetlinks.community.elastic.search.utils.QueryParamTranslator;
import org.jetlinks.core.utils.FluxUtils;
import org.reactivestreams.Publisher;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.DependsOn;
@ -57,6 +57,8 @@ import java.util.regex.Pattern;
import java.util.stream.Collectors;
/**
* 响应式ES数据库操作类
*
* @author zhouhao
* @since 1.0
**/
@ -66,20 +68,23 @@ import java.util.stream.Collectors;
@ConfigurationProperties(prefix = "elasticsearch")
public class ReactiveElasticSearchService implements ElasticSearchService {
private final ReactiveElasticsearchClient restClient;
private final ElasticSearchIndexManager indexManager;
private FluxSink<Buffer> sink;
public static final IndicesOptions indexOptions = IndicesOptions.fromOptions(
true, true, false, false
);
//使用对象池处理Buffer,减少GC消耗
static ObjectPool<Buffer> pool = ObjectPool.newPool(Buffer::new);
static {
DateFormatter.supportFormatter.add(new DefaultDateFormatter(Pattern.compile("[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.+"), "yyyy-MM-dd'T'HH:mm:ss.SSSZ"));
}
private final ReactiveElasticsearchClient restClient;
private final ElasticSearchIndexManager indexManager;
private FluxSink<Buffer> sink;
@Getter
@Setter
private BufferConfig buffer = new BufferConfig();
public ReactiveElasticSearchService(ReactiveElasticsearchClient restClient,
ElasticSearchIndexManager indexManager) {
this.restClient = restClient;
@ -119,12 +124,14 @@ public class ReactiveElasticSearchService implements ElasticSearchService {
});
}
@Override
public <T> Flux<T> query(String index, QueryParam queryParam, Function<Map<String, Object>, T> mapper) {
return this
.doQuery(new String[]{index}, queryParam)
.flatMapMany(tp2 -> convertQueryResult(tp2.getT1(), tp2.getT2(), mapper));
}
@Override
public <T> Flux<T> query(String[] index, QueryParam queryParam, Function<Map<String, Object>, T> mapper) {
return this
.doQuery(index, queryParam)
@ -184,7 +191,6 @@ public class ReactiveElasticSearchService implements ElasticSearchService {
});
}
@Override
public Mono<Long> count(String[] index, QueryParam queryParam) {
QueryParam param = queryParam.clone();
@ -251,32 +257,6 @@ public class ReactiveElasticSearchService implements ElasticSearchService {
sink.complete();
}
@Getter
@Setter
private BufferConfig buffer = new BufferConfig();
@Getter
@Setter
public static class BufferConfig {
//最小间隔
private int rate = Integer.getInteger("elasticsearch.buffer.rate", 1000);
//缓冲最大数量
private int bufferSize = Integer.getInteger("elasticsearch.buffer.size", 3000);
//缓冲超时时间
private Duration bufferTimeout = Duration.ofSeconds(Integer.getInteger("elasticsearch.buffer.timeout", 3));
//背压堆积数量限制.
private int bufferBackpressure = Integer.getInteger("elasticsearch.buffer.backpressure", Runtime
.getRuntime()
.availableProcessors());
//最大缓冲字节
private DataSize bufferBytes = DataSize.parse(System.getProperty("elasticsearch.buffer.bytes", "15MB"));
//最大重试次数
private int maxRetry = 3;
//重试间隔
private Duration minBackoff = Duration.ofSeconds(3);
}
//@PostConstruct
public void init() {
int flushRate = buffer.rate;
@ -323,52 +303,6 @@ public class ReactiveElasticSearchService implements ElasticSearchService {
.subscribe();
}
//使用对象池处理Buffer,减少GC消耗
static ObjectPool<Buffer> pool = ObjectPool.newPool(Buffer::new);
@Getter
static class Buffer {
String index;
String id;
String payload;
final ObjectPool.Handle<Buffer> handle;
public Buffer(ObjectPool.Handle<Buffer> handle) {
this.handle = handle;
}
public static Buffer of(String index, Object payload) {
Buffer buffer;
try {
buffer = pool.get();
} catch (Exception e) {
buffer = new Buffer(null);
}
buffer.index = index;
Map<String, Object> data = payload instanceof Map
? ((Map) payload) :
FastBeanCopier.copy(payload, HashMap::new);
Object id = data.get("id");
buffer.id = id == null ? null : String.valueOf(id);
buffer.payload = JSON.toJSONString(data);
return buffer;
}
void release() {
this.index = null;
this.id = null;
this.payload = null;
if (null != handle) {
handle.recycle(this);
}
}
int numberOfBytes() {
return payload == null ? 0 : payload.length() * 2;
}
}
private Mono<String> getIndexForSave(String index) {
return indexManager
.getIndexStrategy(index)
@ -516,4 +450,68 @@ public class ReactiveElasticSearchService implements ElasticSearchService {
.filter(CollectionUtils::isNotEmpty)
.flatMap(list -> createCountRequest(queryParam, list));
}
@Getter
@Setter
public static class BufferConfig {
//最小间隔
private int rate = Integer.getInteger("elasticsearch.buffer.rate", 1000);
//缓冲最大数量
private int bufferSize = Integer.getInteger("elasticsearch.buffer.size", 3000);
//缓冲超时时间
private Duration bufferTimeout = Duration.ofSeconds(Integer.getInteger("elasticsearch.buffer.timeout", 3));
//背压堆积数量限制.
private int bufferBackpressure = Integer.getInteger("elasticsearch.buffer.backpressure", Runtime
.getRuntime()
.availableProcessors());
//最大缓冲字节
private DataSize bufferBytes = DataSize.parse(System.getProperty("elasticsearch.buffer.bytes", "15MB"));
//最大重试次数
private int maxRetry = 3;
//重试间隔
private Duration minBackoff = Duration.ofSeconds(3);
}
@Getter
static class Buffer {
final ObjectPool.Handle<Buffer> handle;
String index;
String id;
String payload;
public Buffer(ObjectPool.Handle<Buffer> handle) {
this.handle = handle;
}
public static Buffer of(String index, Object payload) {
Buffer buffer;
try {
buffer = pool.get();
} catch (Exception e) {
buffer = new Buffer(null);
}
buffer.index = index;
Map<String, Object> data = payload instanceof Map
? ((Map) payload) :
FastBeanCopier.copy(payload, HashMap::new);
Object id = data.get("id");
buffer.id = id == null ? null : String.valueOf(id);
buffer.payload = JSON.toJSONString(data);
return buffer;
}
void release() {
this.index = null;
this.id = null;
this.payload = null;
if (null != handle) {
handle.recycle(this);
}
}
int numberOfBytes() {
return payload == null ? 0 : payload.length() * 2;
}
}
}

28
jetlinks-components/gateway-component/src/main/java/org/jetlinks/community/gateway/supports/DefaultDeviceGatewayManager.java
View File

@ -12,23 +12,47 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
/**
* 设备网关管理器
* <p>
* TCP UDP MQTT CoAP
*
* @author zhouhao
*/
@Component
public class DefaultDeviceGatewayManager implements DeviceGatewayManager, BeanPostProcessor {
private final DeviceGatewayPropertiesManager propertiesManager;
private Map<String, DeviceGatewayProvider> providers = new ConcurrentHashMap<>();
/**
* TCP MQTT的设备网关服务提供者
*/
private final Map<String, DeviceGatewayProvider> providers = new ConcurrentHashMap<>();
private Map<String, DeviceGateway> store = new ConcurrentHashMap<>();
/**
* 启动状态的设备网关
*/
private final Map<String, DeviceGateway> store = new ConcurrentHashMap<>();
public DefaultDeviceGatewayManager(DeviceGatewayPropertiesManager propertiesManager) {
this.propertiesManager = propertiesManager;
}
/**
* 获取设备网关，有则返回，没有就创建返回
*
* @param id 网关ID
* @return 设备网关
*/
private Mono<DeviceGateway> doGetGateway(String id) {
if (store.containsKey(id)) {
return Mono.just(store.get(id));
}
// 数据库查 DeviceGatewayEntity 转换成 DeviceGatewayProperties
// BeanMap中找provider 找不到就是不支持
// 创建设备网关
// double check 防止重复创建
return propertiesManager
.getProperties(id)
.switchIfEmpty(Mono.error(() -> new UnsupportedOperationException("网关配置[" + id + "]不存在")))

8
jetlinks-components/gateway-component/src/main/java/org/jetlinks/community/gateway/supports/DeviceGatewayProperties.java
View File

@ -7,6 +7,14 @@ import org.jetlinks.community.ValueObject;
import java.util.HashMap;
import java.util.Map;
/**
* 设备网关属性外观类
* <p>
* 转换设备网关属性数据
* </p>
*
* @author zhouhao
*/
@Getter
@Setter
public class DeviceGatewayProperties implements ValueObject {

11
jetlinks-components/gateway-component/src/main/java/org/jetlinks/community/gateway/supports/DeviceGatewayPropertiesManager.java
View File

@ -2,8 +2,19 @@ package org.jetlinks.community.gateway.supports;
import reactor.core.publisher.Mono;
/**
* 设备网关属性管理器
*
* @author zhouhao
*/
public interface DeviceGatewayPropertiesManager {
/**
* 获取网关的属性
*
* @param id 网关ID
* @return 网关属性
*/
Mono<DeviceGatewayProperties> getProperties(String id);

8
jetlinks-components/gateway-component/src/main/java/org/jetlinks/community/gateway/supports/DeviceGatewayProvider.java
View File

@ -4,6 +4,14 @@ import org.jetlinks.community.gateway.DeviceGateway;
import org.jetlinks.community.network.NetworkType;
import reactor.core.publisher.Mono;
/**
* 设备网关支持提供商,用于提供对各种设备网关的支持.在启动设备网关时,会根据对应的提供商以及配置来创建设备网关.
* 实现统一管理网关配置,动态创建设备网关.
*
* @author zhouhao
* @see DeviceGateway
* @since 1.0
*/
public interface DeviceGatewayProvider {
String getId();

9
jetlinks-components/network-component/network-core/src/main/java/org/jetlinks/community/network/DefaultNetworkManager.java
View File

@ -19,6 +19,11 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
/**
* 默认网络管理器
*
* @author zhouhao
*/
@Component
@Slf4j
public class DefaultNetworkManager implements NetworkManager, BeanPostProcessor {
@ -26,9 +31,9 @@ public class DefaultNetworkManager implements NetworkManager, BeanPostProcessor
private final NetworkConfigManager configManager;
private Map<String, Map<String, Network>> store = new ConcurrentHashMap<>();
private final Map<String, Map<String, Network>> store = new ConcurrentHashMap<>();
private Map<String, NetworkProvider<Object>> providerSupport = new ConcurrentHashMap<>();
private final Map<String, NetworkProvider<Object>> providerSupport = new ConcurrentHashMap<>();
public DefaultNetworkManager(NetworkConfigManager configManager) {
this.configManager = configManager;

35
jetlinks-components/network-component/network-core/src/main/java/org/jetlinks/community/network/NetworkManager.java
View File

@ -4,13 +4,48 @@ import reactor.core.publisher.Mono;
import java.util.List;
/**
* 网络服务管理器
* <p>
* 管理所有的网络组件
*
* @author zhouhao
* @since 1.0
*/
public interface NetworkManager {
/**
* 根据ID获取网络组件，否则根据type和id创建网络组件并返回
*
* @param type 网络类型
* @param id 网络组件id
* @param <T> NetWork子类泛型
* @return 网络组件
*/
<T extends Network> Mono<T> getNetwork(NetworkType type, String id);
/**
* 获取所有的网络组件支持提供商
*
* @return 网络组件支持提供商
*/
List<NetworkProvider<?>> getProviders();
/**
* 重新加载网络组件
*
* @param type 网络类型
* @param id 网络组件ID
* @return void
*/
Mono<Void> reload(NetworkType type, String id);
/**
* 停止网络组件
*
* @param type 网络类型
* @param id 网络组件ID
* @return void
*/
Mono<Void> shutdown(NetworkType type, String id);
}

17
jetlinks-components/network-component/network-core/src/main/java/org/jetlinks/community/network/utils/DeviceGatewayHelper.java
View File

@ -21,6 +21,14 @@ import java.util.function.Consumer;
import java.util.function.Function;
import java.util.function.Supplier;
/**
* 设备网关处理工具
* <p>
* 封装常用的设备消息处理操作
* </p>
*
* @author zhouhao
*/
@AllArgsConstructor
public class DeviceGatewayHelper {
@ -102,6 +110,15 @@ public class DeviceGatewayHelper {
return Mono.empty();
}
/**
* 处理来自设备网关的设备消息
*
* @param message 设备消息
* @param sessionBuilder 设备操作
* @param sessionConsumer 设备消费
* @param deviceNotFoundListener 异常监听
* @return 设备操作
*/
public Mono<DeviceOperator> handleDeviceMessage(DeviceMessage message,
Function<DeviceOperator, DeviceSession> sessionBuilder,
Consumer<DeviceSession> sessionConsumer,

293
jetlinks-components/network-component/tcp-component/src/main/java/org/jetlinks/community/network/tcp/device/TcpServerDeviceGateway.java
View File

@ -3,6 +3,16 @@ package org.jetlinks.community.network.tcp.device;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.hswebframework.web.logger.ReactiveLogger;
import org.jetlinks.community.gateway.DeviceGateway;
import org.jetlinks.community.gateway.monitor.DeviceGatewayMonitor;
import org.jetlinks.community.gateway.monitor.GatewayMonitors;
import org.jetlinks.community.gateway.monitor.MonitorSupportDeviceGateway;
import org.jetlinks.community.network.DefaultNetworkType;
import org.jetlinks.community.network.NetworkType;
import org.jetlinks.community.network.tcp.TcpMessage;
import org.jetlinks.community.network.tcp.client.TcpClient;
import org.jetlinks.community.network.tcp.server.TcpServer;
import org.jetlinks.community.network.utils.DeviceGatewayHelper;
import org.jetlinks.core.ProtocolSupport;
import org.jetlinks.core.ProtocolSupports;
import org.jetlinks.core.device.DeviceOperator;
@ -17,16 +27,6 @@ import org.jetlinks.core.message.codec.Transport;
import org.jetlinks.core.server.DeviceGatewayContext;
import org.jetlinks.core.server.session.DeviceSession;
import org.jetlinks.core.server.session.DeviceSessionManager;
import org.jetlinks.community.gateway.DeviceGateway;
import org.jetlinks.community.gateway.monitor.DeviceGatewayMonitor;
import org.jetlinks.community.gateway.monitor.GatewayMonitors;
import org.jetlinks.community.gateway.monitor.MonitorSupportDeviceGateway;
import org.jetlinks.community.network.DefaultNetworkType;
import org.jetlinks.community.network.NetworkType;
import org.jetlinks.community.network.tcp.TcpMessage;
import org.jetlinks.community.network.tcp.client.TcpClient;
import org.jetlinks.community.network.tcp.server.TcpServer;
import org.jetlinks.community.network.utils.DeviceGatewayHelper;
import org.jetlinks.supports.server.DecodedClientMessageHandler;
import reactor.core.Disposable;
import reactor.core.publisher.EmitterProcessor;
@ -48,6 +48,9 @@ class TcpServerDeviceGateway implements DeviceGateway, MonitorSupportDeviceGatew
@Getter
private final String id;
/**
* 维护所有创建的tcp server
*/
private final TcpServer tcpServer;
private final String protocol;
@ -60,6 +63,9 @@ class TcpServerDeviceGateway implements DeviceGateway, MonitorSupportDeviceGatew
private final DeviceGatewayMonitor gatewayMonitor;
/**
* 连接计数器
*/
private final LongAdder counter = new LongAdder();
private final EmitterProcessor<Message> processor = EmitterProcessor.create(false);
@ -67,10 +73,11 @@ class TcpServerDeviceGateway implements DeviceGateway, MonitorSupportDeviceGatew
private final FluxSink<Message> sink = processor.sink(FluxSink.OverflowStrategy.BUFFER);
private final AtomicBoolean started = new AtomicBoolean();
private Disposable disposable;
private final DeviceGatewayHelper helper;
/**
* 数据流控开关
*/
private Disposable disposable;
public TcpServerDeviceGateway(String id,
String protocol,
@ -93,133 +100,39 @@ class TcpServerDeviceGateway implements DeviceGateway, MonitorSupportDeviceGatew
return supports.getProtocol(protocol);
}
/**
* 当前总链接
*
* @return 当前总链接
*/
@Override
public long totalConnection() {
return counter.sum();
}
/**
* 传输协议
*
* @return {@link org.jetlinks.core.message.codec.DefaultTransport}
*/
@Override
public Transport getTransport() {
return DefaultTransport.TCP;
}
/**
* 网络类型
*
* @return {@link org.jetlinks.community.network.DefaultNetworkType}
*/
@Override
public NetworkType getNetworkType() {
return DefaultNetworkType.TCP_SERVER;
}
class TcpConnection implements DeviceGatewayContext {
final TcpClient client;
final AtomicReference<Duration> keepaliveTimeout = new AtomicReference<>();
final AtomicReference<DeviceSession> sessionRef = new AtomicReference<>();
final InetSocketAddress address;
TcpConnection(TcpClient client) {
this.client = client;
this.address = client.getRemoteAddress();
gatewayMonitor.totalConnection(counter.sum());
client.onDisconnect(() -> {
counter.decrement();
gatewayMonitor.disconnected();
gatewayMonitor.totalConnection(counter.sum());
});
gatewayMonitor.connected();
DeviceSession session = sessionManager.getSession(client.getId());
if (session == null) {
session = new UnknownTcpDeviceSession(client.getId(), client, getTransport()) {
@Override
public Mono<Boolean> send(EncodedMessage encodedMessage) {
return super.send(encodedMessage).doOnSuccess(r -> gatewayMonitor.sentMessage());
}
@Override
public void setKeepAliveTimeout(Duration timeout) {
keepaliveTimeout.set(timeout);
client.setKeepAliveTimeout(timeout);
}
@Override
public Optional<InetSocketAddress> getClientAddress() {
return Optional.of(address);
}
};
}
sessionRef.set(session);
}
Mono<Void> accept() {
return getProtocol()
.flatMap(protocol -> protocol.onClientConnect(getTransport(), client, this))
.then(
client
.subscribe()
.filter(tcp -> started.get())
.publishOn(Schedulers.parallel())
.flatMap(this::handleTcpMessage)
.onErrorResume((err) -> {
log.error(err.getMessage(), err);
client.shutdown();
return Mono.empty();
})
.then()
)
.doOnCancel(client::shutdown);
}
Mono<Void> handleTcpMessage(TcpMessage message) {
return getProtocol()
.flatMap(pt -> pt.getMessageCodec(getTransport()))
.flatMapMany(codec -> codec.decode(FromDeviceMessageContext.of(sessionRef.get(), message, registry)))
.cast(DeviceMessage.class)
.doOnNext(msg -> gatewayMonitor.receivedMessage())
.flatMap(this::handleDeviceMessage)
.doOnEach(ReactiveLogger.onError(err -> log.error("处理TCP[{}]消息失败:\n{}",
address,
message
, err)))
.onErrorResume((err) -> Mono.fromRunnable(client::reset))
.then();
}
Mono<Void> handleDeviceMessage(DeviceMessage message) {
if (processor.hasDownstreams()) {
sink.next(message);
}
return helper
.handleDeviceMessage(message,
device -> new TcpDeviceSession(device, client, getTransport(), gatewayMonitor),
DeviceGatewayHelper
.applySessionKeepaliveTimeout(message, keepaliveTimeout::get)
.andThen(session -> {
TcpDeviceSession deviceSession = session.unwrap(TcpDeviceSession.class);
deviceSession.setClient(client);
sessionRef.set(deviceSession);
}),
() -> log.warn("无法从tcp[{}]消息中获取设备信息:{}", address, message)
)
.then();
}
@Override
public Mono<DeviceOperator> getDevice(String deviceId) {
return registry.getDevice(deviceId);
}
@Override
public Mono<DeviceProductOperator> getProduct(String productId) {
return registry.getProduct(productId);
}
@Override
public Mono<Void> onMessage(DeviceMessage message) {
return handleDeviceMessage(message);
}
}
/**
* 启动网关
*/
private void doStart() {
if (started.getAndSet(true) || disposable != null) {
return;
@ -265,4 +178,134 @@ class TcpServerDeviceGateway implements DeviceGateway, MonitorSupportDeviceGatew
public boolean isAlive() {
return started.get();
}
/**
* TCP 客户端连接
*/
class TcpConnection implements DeviceGatewayContext {
final TcpClient client;
final AtomicReference<Duration> keepaliveTimeout = new AtomicReference<>();
final AtomicReference<DeviceSession> sessionRef = new AtomicReference<>();
final InetSocketAddress address;
TcpConnection(TcpClient client) {
this.client = client;
this.address = client.getRemoteAddress();
gatewayMonitor.totalConnection(counter.sum());
client.onDisconnect(() -> {
counter.decrement();
gatewayMonitor.disconnected();
gatewayMonitor.totalConnection(counter.sum());
});
gatewayMonitor.connected();
DeviceSession session = sessionManager.getSession(client.getId());
if (session == null) {
session = new UnknownTcpDeviceSession(client.getId(), client, getTransport()) {
@Override
public Mono<Boolean> send(EncodedMessage encodedMessage) {
return super.send(encodedMessage).doOnSuccess(r -> gatewayMonitor.sentMessage());
}
@Override
public void setKeepAliveTimeout(Duration timeout) {
keepaliveTimeout.set(timeout);
client.setKeepAliveTimeout(timeout);
}
@Override
public Optional<InetSocketAddress> getClientAddress() {
return Optional.of(address);
}
};
}
sessionRef.set(session);
}
/**
* 接收消息
*
* @return void
*/
Mono<Void> accept() {
return getProtocol()
.flatMap(protocol -> protocol.onClientConnect(getTransport(), client, this))
.then(
client
.subscribe()
.filter(tcp -> started.get())
.publishOn(Schedulers.parallel())
.flatMap(this::handleTcpMessage)
.onErrorResume((err) -> {
log.error(err.getMessage(), err);
client.shutdown();
return Mono.empty();
})
.then()
)
.doOnCancel(client::shutdown);
}
/**
* 处理TCP消息 ==>> 设备消息
*
* @param message tcp消息
* @return void
*/
Mono<Void> handleTcpMessage(TcpMessage message) {
return getProtocol()
.flatMap(pt -> pt.getMessageCodec(getTransport()))
.flatMapMany(codec -> codec.decode(FromDeviceMessageContext.of(sessionRef.get(), message, registry)))
.cast(DeviceMessage.class)
.doOnNext(msg -> gatewayMonitor.receivedMessage())
.flatMap(this::handleDeviceMessage)
.doOnEach(ReactiveLogger.onError(err -> log.error("处理TCP[{}]消息失败:\n{}",
address,
message
, err)))
.onErrorResume((err) -> Mono.fromRunnable(client::reset))
.then();
}
/**
* 处理设备消息
*
* @param message 设备消息
* @return void
*/
Mono<Void> handleDeviceMessage(DeviceMessage message) {
if (processor.hasDownstreams()) {
sink.next(message);
}
return helper
.handleDeviceMessage(message,
device -> new TcpDeviceSession(device, client, getTransport(), gatewayMonitor),
DeviceGatewayHelper
.applySessionKeepaliveTimeout(message, keepaliveTimeout::get)
.andThen(session -> {
TcpDeviceSession deviceSession = session.unwrap(TcpDeviceSession.class);
deviceSession.setClient(client);
sessionRef.set(deviceSession);
}),
() -> log.warn("无法从tcp[{}]消息中获取设备信息:{}", address, message)
)
.then();
}
@Override
public Mono<DeviceOperator> getDevice(String deviceId) {
return registry.getDevice(deviceId);
}
@Override
public Mono<DeviceProductOperator> getProduct(String productId) {
return registry.getProduct(productId);
}
@Override
public Mono<Void> onMessage(DeviceMessage message) {
return handleDeviceMessage(message);
}
}
}

12
jetlinks-components/network-component/tcp-component/src/main/java/org/jetlinks/community/network/tcp/device/TcpServerDeviceGatewayProvider.java
View File

@ -1,8 +1,5 @@
package org.jetlinks.community.network.tcp.device;
import org.jetlinks.core.ProtocolSupports;
import org.jetlinks.core.device.DeviceRegistry;
import org.jetlinks.core.server.session.DeviceSessionManager;
import org.jetlinks.community.gateway.DeviceGateway;
import org.jetlinks.community.gateway.supports.DeviceGatewayProperties;
import org.jetlinks.community.gateway.supports.DeviceGatewayProvider;
@ -10,11 +7,20 @@ import org.jetlinks.community.network.DefaultNetworkType;
import org.jetlinks.community.network.NetworkManager;
import org.jetlinks.community.network.NetworkType;
import org.jetlinks.community.network.tcp.server.TcpServer;
import org.jetlinks.core.ProtocolSupports;
import org.jetlinks.core.device.DeviceRegistry;
import org.jetlinks.core.server.session.DeviceSessionManager;
import org.jetlinks.supports.server.DecodedClientMessageHandler;
import org.springframework.stereotype.Component;
import org.springframework.util.Assert;
import reactor.core.publisher.Mono;
/**
* TCP服务设备网关提供商
*
* @author zhouhao
* @since 1.0
*/
@Component
public class TcpServerDeviceGatewayProvider implements DeviceGatewayProvider {

1
jetlinks-components/network-component/tcp-component/src/main/java/org/jetlinks/community/network/tcp/server/TcpServer.java
View File

@ -23,5 +23,6 @@ public interface TcpServer extends Network {
/**
* 关闭服务端
*/
@Override
void shutdown();
}

22
jetlinks-components/timeseries-component/src/main/java/org/jetlinks/community/timeseries/TimeSeriesData.java
View File

@ -7,8 +7,21 @@ import java.util.Date;
import java.util.Map;
import java.util.Optional;
/**
* 时序数据封装类
*
* @author zhouhao
*/
public interface TimeSeriesData extends ValueObject {
static TimeSeriesData of(Date date, Map<String, Object> data) {
return of(date == null ? System.currentTimeMillis() : date.getTime(), data);
}
static TimeSeriesData of(long timestamp, Map<String, Object> data) {
return new SimpleTimeSeriesData(timestamp, data);
}
long getTimestamp();
Map<String, Object> getData();
@ -23,14 +36,7 @@ public interface TimeSeriesData extends ValueObject {
return Optional.ofNullable(getData().get(name));
}
static TimeSeriesData of(Date date, Map<String, Object> data) {
return of(date == null ? System.currentTimeMillis() : date.getTime(), data);
}
static TimeSeriesData of(long timestamp, Map<String, Object> data) {
return new SimpleTimeSeriesData(timestamp, data);
}
@Override
default <T> T as(Class<T> type) {
return FastBeanCopier.copy(getData(), type);
}

230
jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/message/DeviceMessageConnector.java
View File

@ -36,115 +36,9 @@ public class DeviceMessageConnector implements DecodedClientMessageHandler {
PropertyConstants.deviceName.getKey(),
PropertyConstants.orgId.getKey()
};
//设备注册中心
private final DeviceRegistry registry;
private final EventBus eventBus;
private final MessageHandler messageHandler;
private final static BiConsumer<Throwable, Object> doOnError = (error, val) -> DeviceMessageConnector.log.error(error.getMessage(), error);
private final static Function<DeviceOperator, Mono<Values>> configGetter = operator -> operator.getSelfConfigs(allConfigHeader);
private final static Values emptyValues = Values.of(Collections.emptyMap());
public DeviceMessageConnector(EventBus eventBus,
DeviceRegistry registry,
MessageHandler messageHandler,
DeviceSessionManager sessionManager) {
this.registry = registry;
this.eventBus = eventBus;
this.messageHandler = messageHandler;
sessionManager
.onRegister()
.flatMap(session -> {
DeviceOnlineMessage message = new DeviceOnlineMessage();
message.setDeviceId(session.getDeviceId());
message.setTimestamp(session.connectTime());
return onMessage(message);
})
.onErrorContinue(doOnError)
.subscribe();
sessionManager
.onUnRegister()
.flatMap(session -> {
DeviceOfflineMessage message = new DeviceOfflineMessage();
message.setDeviceId(session.getDeviceId());
message.setTimestamp(System.currentTimeMillis());
return onMessage(message);
})
.onErrorContinue(doOnError)
.subscribe();
}
public Mono<Void> onMessage(Message message) {
if (null == message) {
return Mono.empty();
}
return this
.getTopic(message)
.flatMap(topic -> eventBus.publish(topic, message).then())
.onErrorContinue(doOnError)
.then();
}
private Flux<String> getTopic(Message message) {
Flux<String> topicsStream = createDeviceMessageTopic(registry, message);
if (message instanceof ChildDeviceMessage) { //子设备消息
return this
.onMessage(((ChildDeviceMessage) message).getChildDeviceMessage())
.thenMany(topicsStream);
} else if (message instanceof ChildDeviceMessageReply) { //子设备消息
return this
.onMessage(((ChildDeviceMessageReply) message).getChildDeviceMessage())
.thenMany(topicsStream);
}
return topicsStream;
}
public static Flux<String> createDeviceMessageTopic(DeviceRegistry deviceRegistry, Message message) {
return Flux.defer(() -> {
if (message instanceof DeviceMessage) {
DeviceMessage deviceMessage = ((DeviceMessage) message);
String deviceId = deviceMessage.getDeviceId();
if (deviceId == null) {
log.warn("无法从消息中获取设备ID:{}", deviceMessage);
return Mono.empty();
}
return deviceRegistry
.getDevice(deviceId)
.flatMap(configGetter)
.defaultIfEmpty(emptyValues)
.flatMapIterable(configs -> {
configs.getAllValues().forEach(deviceMessage::addHeader);
String productId = deviceMessage.getHeader(PropertyConstants.productId).orElse("null");
String topic = createDeviceMessageTopic(productId, deviceId, deviceMessage);
List<String> topics = new ArrayList<>(2);
topics.add(topic);
configs.getValue(PropertyConstants.orgId)
.ifPresent(orgId -> topics.add("/org/" + orgId + topic));
return topics;
});
}
return Mono.just("/device/unknown/message/unknown");
});
}
public static String createDeviceMessageTopic(String productId, String deviceId, DeviceMessage message) {
StringBuilder builder = new StringBuilder(64)
.append("/device/")
.append(productId)
.append("/")
.append(deviceId);
appendDeviceMessageTopic(message, builder);
return builder.toString();
}
private static final BiConsumer<Message, StringBuilder>[] fastTopicBuilder;
static {
@ -229,6 +123,80 @@ public class DeviceMessageConnector implements DecodedClientMessageHandler {
createFastBuilder(MessageType.DERIVED_METADATA, "/metadata/derived");
}
private final DeviceRegistry registry;
private final EventBus eventBus;
private final MessageHandler messageHandler;
public DeviceMessageConnector(EventBus eventBus,
DeviceRegistry registry,
MessageHandler messageHandler,
DeviceSessionManager sessionManager) {
this.registry = registry;
this.eventBus = eventBus;
this.messageHandler = messageHandler;
sessionManager
.onRegister()
.flatMap(session -> {
DeviceOnlineMessage message = new DeviceOnlineMessage();
message.setDeviceId(session.getDeviceId());
message.setTimestamp(session.connectTime());
return onMessage(message);
})
.onErrorContinue(doOnError)
.subscribe();
sessionManager
.onUnRegister()
.flatMap(session -> {
DeviceOfflineMessage message = new DeviceOfflineMessage();
message.setDeviceId(session.getDeviceId());
message.setTimestamp(System.currentTimeMillis());
return onMessage(message);
})
.onErrorContinue(doOnError)
.subscribe();
}
public static Flux<String> createDeviceMessageTopic(DeviceRegistry deviceRegistry, Message message) {
return Flux.defer(() -> {
if (message instanceof DeviceMessage) {
DeviceMessage deviceMessage = ((DeviceMessage) message);
String deviceId = deviceMessage.getDeviceId();
if (deviceId == null) {
log.warn("无法从消息中获取设备ID:{}", deviceMessage);
return Mono.empty();
}
return deviceRegistry
.getDevice(deviceId)
.flatMap(configGetter)
.defaultIfEmpty(emptyValues)
.flatMapIterable(configs -> {
configs.getAllValues().forEach(deviceMessage::addHeader);
String productId = deviceMessage.getHeader(PropertyConstants.productId).orElse("null");
String topic = createDeviceMessageTopic(productId, deviceId, deviceMessage);
List<String> topics = new ArrayList<>(2);
topics.add(topic);
configs.getValue(PropertyConstants.orgId)
.ifPresent(orgId -> topics.add("/org/" + orgId + topic));
return topics;
});
}
return Mono.just("/device/unknown/message/unknown");
});
}
public static String createDeviceMessageTopic(String productId, String deviceId, DeviceMessage message) {
StringBuilder builder = new StringBuilder(64)
.append("/device/")
.append(productId)
.append("/")
.append(deviceId);
appendDeviceMessageTopic(message, builder);
return builder.toString();
}
private static void createFastBuilder(MessageType messageType,
String topic) {
fastTopicBuilder[messageType.ordinal()] = (ignore, builder) -> builder.append(topic);
@ -249,6 +217,37 @@ public class DeviceMessageConnector implements DecodedClientMessageHandler {
}
}
public Mono<Void> onMessage(Message message) {
if (null == message) {
return Mono.empty();
}
return this
.getTopic(message)
.flatMap(topic -> eventBus.publish(topic, message).then())
.onErrorContinue(doOnError)
.then();
}
private Flux<String> getTopic(Message message) {
Flux<String> topicsStream = createDeviceMessageTopic(registry, message);
if (message instanceof ChildDeviceMessage) { //子设备消息
return this
.onMessage(((ChildDeviceMessage) message).getChildDeviceMessage())
.thenMany(topicsStream);
} else if (message instanceof ChildDeviceMessageReply) { //子设备消息
return this
.onMessage(((ChildDeviceMessageReply) message).getChildDeviceMessage())
.thenMany(topicsStream);
}
return topicsStream;
}
/**
* 处理设备消息
*
* @param message 设备消息
* @return 处理结果
*/
protected Mono<Boolean> handleChildrenDeviceMessage(Message message) {
if (message instanceof DeviceMessageReply) {
return doReply(((DeviceMessageReply) message));
@ -261,10 +260,23 @@ public class DeviceMessageConnector implements DecodedClientMessageHandler {
return handleChildrenDeviceMessage(reply.getChildDeviceMessage());
}
/**
* 处理回复消息
*
* @param reply 子设备回复消息
* @return 处理结果
*/
protected Mono<Boolean> handleChildrenDeviceMessageReply(ChildDeviceMessageReply reply) {
return handleChildrenDeviceMessage(reply.getChildDeviceMessage());
}
/**
* 这里才是真正处理消息的地方
*
* @param device 设备操作类
* @param message 设备消息
* @return 处理结果
*/
@Override
public Mono<Boolean> handleMessage(DeviceOperator device, @Nonnull Message message) {
Mono<Boolean> then;
@ -284,6 +296,12 @@ public class DeviceMessageConnector implements DecodedClientMessageHandler {
}
/**
* 回复消息处理逻辑
*
* @param reply 设备回复消息
* @return 处理结果
*/
private Mono<Boolean> doReply(DeviceMessageReply reply) {
if (log.isDebugEnabled()) {
log.debug("reply message {}", reply.getMessageId());

6
jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/message/writer/TimeSeriesMessageWriterConnector.java
View File

@ -19,6 +19,12 @@ public class TimeSeriesMessageWriterConnector{
private final DeviceDataService dataService;
/**
* 订阅设备消息 入库
*
* @param message 设备消息
* @return void
*/
@Subscribe(topics = "/device/**", id = "device-message-ts-writer")
public Mono<Void> writeDeviceMessageToTs(DeviceMessage message) {
return dataService.saveDeviceMessage(message);

77
jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/service/data/AbstractDeviceDataStoragePolicy.java
View File

@ -8,28 +8,22 @@ import org.hswebframework.ezorm.core.param.TermType;
import org.hswebframework.web.api.crud.entity.PagerResult;
import org.hswebframework.web.api.crud.entity.QueryParamEntity;
import org.hswebframework.web.id.IDGenerator;
import org.jetlinks.community.device.entity.DeviceEvent;
import org.jetlinks.community.device.entity.DeviceOperationLogEntity;
import org.jetlinks.community.device.entity.DeviceProperty;
import org.jetlinks.community.device.enums.DeviceLogType;
import org.jetlinks.community.device.events.handler.ValueTypeTranslator;
import org.jetlinks.community.gateway.DeviceMessageUtils;
import org.jetlinks.community.timeseries.TimeSeriesData;
import org.jetlinks.core.device.DeviceConfigKey;
import org.jetlinks.core.device.DeviceProductOperator;
import org.jetlinks.core.device.DeviceRegistry;
import org.jetlinks.core.message.DeviceLogMessage;
import org.jetlinks.core.message.DeviceMessage;
import org.jetlinks.core.message.DeviceMessageReply;
import org.jetlinks.core.message.Headers;
import org.jetlinks.core.message.event.EventMessage;
import org.jetlinks.core.message.property.ReadPropertyMessageReply;
import org.jetlinks.core.message.property.ReportPropertyMessage;
import org.jetlinks.core.message.property.WritePropertyMessageReply;
import org.jetlinks.core.metadata.*;
import org.jetlinks.core.metadata.types.*;
import org.jetlinks.community.device.entity.DeviceEvent;
import org.jetlinks.community.device.entity.DeviceOperationLogEntity;
import org.jetlinks.community.device.entity.DevicePropertiesEntity;
import org.jetlinks.community.device.entity.DeviceProperty;
import org.jetlinks.community.device.enums.DeviceLogType;
import org.jetlinks.community.device.events.handler.ValueTypeTranslator;
import org.jetlinks.community.device.timeseries.DeviceTimeSeriesMetric;
import org.jetlinks.community.timeseries.TimeSeriesData;
import org.jetlinks.core.utils.DeviceMessageTracer;
import org.jetlinks.core.utils.TimestampUtils;
import org.reactivestreams.Publisher;
@ -43,7 +37,6 @@ import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.stream.Collectors;
@ -59,8 +52,8 @@ import static org.jetlinks.community.device.timeseries.DeviceTimeSeriesMetric.*;
*/
public abstract class AbstractDeviceDataStoragePolicy implements DeviceDataStoragePolicy {
private final AtomicInteger nanoInc = new AtomicInteger();
protected DeviceRegistry deviceRegistry;
protected DeviceDataStorageProperties properties;
public AbstractDeviceDataStoragePolicy(DeviceRegistry registry,
@ -88,9 +81,11 @@ public abstract class AbstractDeviceDataStoragePolicy implements DeviceDataStora
protected abstract Mono<Void> doSaveData(String metric, Flux<TimeSeriesData> data);
/**
* 设备消息转换 二元组 {deviceId, tsData}
*
* @param productId 产品ID
* @param message 原始消息
* @param properties 属性
* @param message 设备属性消息
* @param properties 物模型属性
* @return 数据集合
* @see this#convertPropertiesForColumnPolicy(String, DeviceMessage, Map)
* @see this#convertPropertiesForRowPolicy(String, DeviceMessage, Map)
@ -107,7 +102,15 @@ public abstract class AbstractDeviceDataStoragePolicy implements DeviceDataStora
QueryParamEntity paramEntity,
Function<TimeSeriesData, T> mapper);
/**
* 保存单个设备消息,为了提升性能,存储策略会对保存请求进行缓冲,达到一定条件后
* 再进行批量写出,具体由不同对存储策略实现。
* <p>
* 如果保存失败,在这里不会得到错误信息.
*
* @param message 设备消息
* @return void
*/
@Nonnull
@Override
public Mono<Void> saveDeviceMessage(@Nonnull DeviceMessage message) {
@ -152,6 +155,12 @@ public abstract class AbstractDeviceDataStoragePolicy implements DeviceDataStora
.toSimpleMap())));
}
/**
* 设备消息转换成时序数据 二元组 {deviceId, tsData}
*
* @param message 设备消息
* @return 二元组
*/
protected Flux<Tuple2<String, TimeSeriesData>> convertMessageToTimeSeriesData(DeviceMessage message) {
boolean ignoreStorage = message.getHeaderOrDefault(Headers.ignoreStorage);
boolean ignoreLog = message.getHeaderOrDefault(Headers.ignoreLog);
@ -194,8 +203,16 @@ public abstract class AbstractDeviceDataStoragePolicy implements DeviceDataStora
return Flux.merge(all);
}
/**
* 事件消息转换成 二元组{deviceId, tsData}
*
* @param productId 产品ID
* @param message 事件消息
* @return 二元组
*/
protected Mono<Tuple2<String, TimeSeriesData>> convertEventMessageToTimeSeriesData(String productId, EventMessage message) {
// 设备注册中心获取设备操作接口
// 获取设备元数据 物模型
return deviceRegistry
.getDevice(message.getDeviceId())
.flatMap(device -> device
@ -227,7 +244,7 @@ public abstract class AbstractDeviceDataStoragePolicy implements DeviceDataStora
.map(data -> Tuples.of(deviceEventMetricId(productId, message.getEvent()), data));
}
@Override
public Mono<PagerResult<DeviceOperationLogEntity>> queryDeviceMessageLog(@Nonnull String deviceId, @Nonnull QueryParamEntity entity) {
return deviceRegistry
.getDevice(deviceId)
@ -240,7 +257,6 @@ public abstract class AbstractDeviceDataStoragePolicy implements DeviceDataStora
.defaultIfEmpty(PagerResult.empty());
}
@Nonnull
@Override
public Flux<DeviceEvent> queryEvent(@Nonnull String deviceId,
@ -383,6 +399,16 @@ public abstract class AbstractDeviceDataStoragePolicy implements DeviceDataStora
return Maps.newHashMapWithExpectedSize(size);
}
/**
* 设备消息转换 二元组{deviceId, tsData}
*
* @param productId 产品ID
* @param message 设备属性消息
* @param properties 物模型属性
* @return 数据集合
* @see this#convertPropertiesForColumnPolicy(String, DeviceMessage, Map)
* @see this#convertPropertiesForRowPolicy(String, DeviceMessage, Map)
*/
protected Flux<Tuple2<String, TimeSeriesData>> convertPropertiesForRowPolicy(String productId,
DeviceMessage message,
Map<String, Object> properties) {
@ -529,11 +555,12 @@ public abstract class AbstractDeviceDataStoragePolicy implements DeviceDataStora
.collect(Collectors.toList());
}
private final AtomicInteger nanoInc = new AtomicInteger();
//将毫秒转为纳秒，努力让数据不重复
/**
* 将毫秒转为纳秒，努力让数据不重复
*
* @param millis 毫秒值
* @return 尽可能不会重复的long值
*/
protected long createUniqueNanoTime(long millis) {
long nano = TimeUnit.MILLISECONDS.toNanos(millis);

44
jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/service/data/DefaultDeviceDataService.java
View File

@ -2,16 +2,16 @@ package org.jetlinks.community.device.service.data;
import org.hswebframework.web.api.crud.entity.PagerResult;
import org.hswebframework.web.api.crud.entity.QueryParamEntity;
import org.jetlinks.community.device.entity.DeviceEvent;
import org.jetlinks.community.device.entity.DeviceOperationLogEntity;
import org.jetlinks.community.device.entity.DeviceProperty;
import org.jetlinks.community.timeseries.query.AggregationData;
import org.jetlinks.core.Value;
import org.jetlinks.core.device.DeviceOperator;
import org.jetlinks.core.device.DeviceProductOperator;
import org.jetlinks.core.device.DeviceRegistry;
import org.jetlinks.core.message.DeviceMessage;
import org.jetlinks.core.metadata.DeviceMetadata;
import org.jetlinks.community.device.entity.DeviceEvent;
import org.jetlinks.community.device.entity.DeviceOperationLogEntity;
import org.jetlinks.community.device.entity.DeviceProperty;
import org.jetlinks.community.timeseries.query.AggregationData;
import org.reactivestreams.Publisher;
import org.springframework.beans.factory.ObjectProvider;
import org.springframework.stereotype.Component;
@ -23,6 +23,13 @@ import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Function;
/**
* 默认设备数据服务
* <p>
* 管理设备存储策略、提供数据查询和入库操作
*
* @author zhouhao
*/
@Component
public class DefaultDeviceDataService implements DeviceDataService {
@ -55,8 +62,16 @@ public class DefaultDeviceDataService implements DeviceDataService {
.then();
}
/**
* 通过产品ID 获取存储策略
*
* @param productId 产品ID
* @return 存储策略
*/
Mono<DeviceDataStoragePolicy> getStoreStrategy(String productId) {
// 从注册中心获取产品操作接口
// 从配置中获取产品的存储策略
// 巧妙的双层switchIfEmpty 外层判断空配置 内层判断空策略
return deviceRegistry
.getProduct(productId)
.flatMap(product -> product
@ -69,7 +84,16 @@ public class DefaultDeviceDataService implements DeviceDataService {
.flatMap(Function.identity()));
}
/**
* 通过设备ID 获取存储策略
*
* @param deviceId 设备ID
* @return 存储策略
*/
Mono<DeviceDataStoragePolicy> getDeviceStrategy(String deviceId) {
// 从注册中心获取设备操作接口
// 转换成产品操作接口
// 继而通过转换的产品ID获取存储策略
return deviceRegistry.getDevice(deviceId)
.flatMap(DeviceOperator::getProduct)
.map(DeviceProductOperator::getId)
@ -145,7 +169,15 @@ public class DefaultDeviceDataService implements DeviceDataService {
.defaultIfEmpty(PagerResult.empty());
}
/**
* 保存单个设备消息,为了提升性能,存储策略会对保存请求进行缓冲,达到一定条件后
* 再进行批量写出,具体由不同对存储策略实现。
* <p>
* 如果保存失败,在这里不会得到错误信息.
*
* @param message 设备消息
* @return void
*/
@Nonnull
@Override
public Mono<Void> saveDeviceMessage(@Nonnull DeviceMessage message) {

32
jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/service/data/TimeSeriesColumnDeviceDataStoragePolicy.java
View File

@ -2,7 +2,14 @@ package org.jetlinks.community.device.service.data;
import org.hswebframework.web.api.crud.entity.PagerResult;
import org.hswebframework.web.api.crud.entity.QueryParamEntity;
import org.jetlinks.community.timeseries.query.*;
import org.jetlinks.community.device.entity.DeviceProperty;
import org.jetlinks.community.device.timeseries.DeviceTimeSeriesMetadata;
import org.jetlinks.community.timeseries.TimeSeriesData;
import org.jetlinks.community.timeseries.TimeSeriesManager;
import org.jetlinks.community.timeseries.query.AggregationData;
import org.jetlinks.community.timeseries.query.AggregationQueryParam;
import org.jetlinks.community.timeseries.query.Group;
import org.jetlinks.community.timeseries.query.TimeGroup;
import org.jetlinks.core.device.DeviceOperator;
import org.jetlinks.core.device.DeviceRegistry;
import org.jetlinks.core.message.DeviceMessage;
@ -10,10 +17,6 @@ import org.jetlinks.core.metadata.ConfigMetadata;
import org.jetlinks.core.metadata.Converter;
import org.jetlinks.core.metadata.DeviceMetadata;
import org.jetlinks.core.metadata.PropertyMetadata;
import org.jetlinks.community.device.entity.DeviceProperty;
import org.jetlinks.community.device.timeseries.DeviceTimeSeriesMetadata;
import org.jetlinks.community.timeseries.TimeSeriesData;
import org.jetlinks.community.timeseries.TimeSeriesManager;
import org.joda.time.DateTime;
import org.joda.time.format.DateTimeFormat;
import org.springframework.stereotype.Component;
@ -28,8 +31,12 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
import static org.jetlinks.community.device.timeseries.DeviceTimeSeriesMetric.devicePropertyMetric;
import static org.jetlinks.community.device.timeseries.DeviceTimeSeriesMetric.devicePropertyMetricId;
/**
* 时序数据列存储策略
*
* @author zhouhao
*/
@Component
public class TimeSeriesColumnDeviceDataStoragePolicy extends TimeSeriesDeviceDataStoragePolicy implements DeviceDataStoragePolicy {
@ -77,8 +84,6 @@ public class TimeSeriesColumnDeviceDataStoragePolicy extends TimeSeriesDeviceDat
String deviceId,
Map<String, PropertyMetadata> property,
QueryParamEntity param) {
//查询多个属性,分组聚合获取第一条数据
return param
.toQuery()
@ -243,11 +248,22 @@ public class TimeSeriesColumnDeviceDataStoragePolicy extends TimeSeriesDeviceDat
.doOnNext(agg -> agg.values().remove("_time"));
}
/**
* 设备消息转换 二元组{deviceId, tsData}
*
* @param productId 产品ID
* @param message 设备属性消息
* @param properties 物模型属性
* @return 数据集合
* @see this#convertPropertiesForColumnPolicy(String, DeviceMessage, Map)
* @see this#convertPropertiesForRowPolicy(String, DeviceMessage, Map)
*/
@Override
protected Flux<Tuple2<String, TimeSeriesData>> convertProperties(String productId, DeviceMessage message, Map<String, Object> properties) {
return convertPropertiesForColumnPolicy(productId, message, properties);
}
@Override
protected Object convertPropertyValue(Object value, PropertyMetadata metadata) {
if (value == null || metadata == null) {
return value;

12
jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/service/data/TimeSeriesDeviceDataStoragePolicy.java
View File

@ -10,6 +10,14 @@ import reactor.core.publisher.Mono;
import java.util.function.Function;
/**
* 抽象时序数据存储策略
* <p>
* 提供时序数据通用的查询存储逻辑
* </p>
*
* @author zhouhao
*/
public abstract class TimeSeriesDeviceDataStoragePolicy extends AbstractDeviceDataStoragePolicy {
@ -22,18 +30,21 @@ public abstract class TimeSeriesDeviceDataStoragePolicy extends AbstractDeviceDa
this.timeSeriesManager = timeSeriesManager;
}
@Override
protected Mono<Void> doSaveData(String metric, TimeSeriesData data) {
return timeSeriesManager
.getService(metric)
.commit(data);
}
@Override
protected Mono<Void> doSaveData(String metric, Flux<TimeSeriesData> data) {
return timeSeriesManager
.getService(metric)
.save(data);
}
@Override
protected <T> Flux<T> doQuery(String metric,
QueryParamEntity paramEntity,
Function<TimeSeriesData, T> mapper) {
@ -44,6 +55,7 @@ public abstract class TimeSeriesDeviceDataStoragePolicy extends AbstractDeviceDa
}
@Override
protected <T> Mono<PagerResult<T>> doQueryPager(String metric,
QueryParamEntity paramEntity,
Function<TimeSeriesData, T> mapper) {

27
jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/service/data/TimeSeriesRowDeviceDataStoreStoragePolicy.java
View File

@ -2,18 +2,18 @@ package org.jetlinks.community.device.service.data;
import org.hswebframework.web.api.crud.entity.PagerResult;
import org.hswebframework.web.api.crud.entity.QueryParamEntity;
import org.jetlinks.core.device.DeviceOperator;
import org.jetlinks.core.device.DeviceRegistry;
import org.jetlinks.core.message.DeviceMessage;
import org.jetlinks.core.metadata.ConfigMetadata;
import org.jetlinks.core.metadata.DeviceMetadata;
import org.jetlinks.core.metadata.PropertyMetadata;
import org.jetlinks.community.device.entity.DeviceProperty;
import org.jetlinks.community.device.timeseries.DeviceTimeSeriesMetadata;
import org.jetlinks.community.device.timeseries.DeviceTimeSeriesMetric;
import org.jetlinks.community.timeseries.TimeSeriesData;
import org.jetlinks.community.timeseries.TimeSeriesManager;
import org.jetlinks.community.timeseries.query.*;
import org.jetlinks.core.device.DeviceOperator;
import org.jetlinks.core.device.DeviceRegistry;
import org.jetlinks.core.message.DeviceMessage;
import org.jetlinks.core.metadata.ConfigMetadata;
import org.jetlinks.core.metadata.DeviceMetadata;
import org.jetlinks.core.metadata.PropertyMetadata;
import org.jetlinks.reactor.ql.utils.CastUtils;
import org.springframework.stereotype.Component;
import reactor.core.publisher.Flux;
@ -28,6 +28,11 @@ import java.util.stream.Stream;
import static org.jetlinks.community.device.timeseries.DeviceTimeSeriesMetric.devicePropertyMetric;
/**
* 设备时序数据行存储策略
*
* @author zhouhao
*/
@Component
public class TimeSeriesRowDeviceDataStoreStoragePolicy extends TimeSeriesDeviceDataStoragePolicy implements DeviceDataStoragePolicy {
@ -285,6 +290,16 @@ public class TimeSeriesRowDeviceDataStoreStoragePolicy extends TimeSeriesDeviceD
.doOnNext(agg -> agg.values().remove("_time"));
}
/**
* 设备消息转换 二元组{deviceId, tsData}
*
* @param productId 产品ID
* @param message 设备属性消息
* @param properties 物模型属性
* @return 数据集合
* @see this#convertPropertiesForColumnPolicy(String, DeviceMessage, Map)
* @see this#convertPropertiesForRowPolicy(String, DeviceMessage, Map)
*/
@Override
protected Flux<Tuple2<String, TimeSeriesData>> convertProperties(String productId, DeviceMessage message, Map<String, Object> properties) {
return convertPropertiesForRowPolicy(productId, message, properties);

9
jetlinks-manager/device-manager/src/main/java/org/jetlinks/community/device/timeseries/DeviceTimeSeriesMetric.java
View File

@ -1,8 +1,8 @@
package org.jetlinks.community.device.timeseries;
import org.jetlinks.community.timeseries.TimeSeriesMetric;
import org.jetlinks.core.device.DeviceProductOperator;
import org.jetlinks.core.metadata.EventMetadata;
import org.jetlinks.community.timeseries.TimeSeriesMetric;
/**
* 设备时序数据度量标识
@ -26,6 +26,13 @@ public interface DeviceTimeSeriesMetric {
return TimeSeriesMetric.of(deviceEventMetricId(productId, eventId));
}
/**
* 构建事件指标ID
*
* @param productId 产品ID
* @param eventId 事件ID
* @return 事件指标ID
*/
static String deviceEventMetricId(String productId, String eventId) {
return "event_".concat(productId).concat("_").concat(eventId);
}

5
jetlinks-manager/network-manager/src/main/java/org/jetlinks/community/network/manager/service/DeviceGatewayConfigService.java
View File

@ -7,6 +7,11 @@ import org.jetlinks.community.gateway.supports.DeviceGatewayPropertiesManager;
import org.springframework.stereotype.Service;
import reactor.core.publisher.Mono;
/**
* 设备网关配置服务
*
* @author zhouhao
*/
@Service
public class DeviceGatewayConfigService implements DeviceGatewayPropertiesManager {