IoTDB in springboot2 (1) Integration and writing

IoTDB in springboot2 (1) Integration and writing into springboot

Due to the needs of the company's business, it is necessary to record and display equipment-related indicator information.
After trying the practical applications of mysql and MongoDB, I found that it is still too slow in front of a large amount of data.
If MongoDB queries data for several months or half a year, the speed is not very slow.
But if there is paging and the speed still cannot be improved, it may also be an application problem.
Moreover, the pressure on the CPU and memory caused by frequent reading and writing during the practical process is also very huge.
Therefore, after discussion, we decided to conduct research on time series databases and chose IoTDB, a domestic database. Without further ado, let’s go straight to the content, so the following is based on 1.X.

1. maven reference

<dependency>
            <groupId>org.apache.iotdb</groupId>
            <artifactId>iotdb-session</artifactId>
            <version>1.1.0</version>
            <exclusions>
                <exclusion>
                    <artifactId>logback-core</artifactId>
                    <groupId>ch.qos.logback</groupId>
                </exclusion>
                <exclusion>
                    <artifactId>logback-classic</artifactId>
                    <groupId>ch.qos.logback</groupId>
                </exclusion>
            </exclusions>
        </dependency>
Because there are Log-related conflicts, I need to eliminate Logback-related packages from my project. This is decided based on my actual situation.

2. yml configuration

  iotdb:
    username: root
    password: root
    IP: 192.168.1.127
    port: 6667
    maxSize: 100
The default port is 6667

3. Connection pool configuration

 
import org.apache.iotdb.isession.SessionDataSet;
import org.apache.iotdb.isession.util.Version;
import org.apache.iotdb.rpc.IoTDBConnectionException;
import org.apache.iotdb.rpc.StatementExecutionException;
import org.apache.iotdb.session.Session;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.stereotype.Component;
 
import java.util.ArrayList;
import java.util.List;
 
/**
 * IoTDB thread pool
 * @author zgy
 * @date 2023-6
 */
@Component
@Configuration
public class IoTDBSessionConfig {<!-- -->
 
    private static Logger logger = LoggerFactory.getLogger(IoTDBSessionConfig.class);
 
    @Value("${spring.iotdb.username}")
    private String username;
 
    @Value("${spring.iotdb.password}")
    private String password;
 
    @Value("${spring.iotdb.ip}")
    private String ip;
 
    @Value("${spring.iotdb.port}")
    private Integer port;
 
    @Value("${spring.iotdb.maxSize}")
    private Integer maxSize;
 
    private static Session session;
 
    /**
     * initialization
     * @return
     * @throws IoTDBConnectionException
     * @throws StatementExecutionException
     */
    @Bean
    public Session getSession() throws IoTDBConnectionException, StatementExecutionException {<!-- -->
        if (session == null) {<!-- -->
            logger.info("Connecting to iotdb....");
            session = new Session.Builder()
                    .host(ip)
                    .port(port)
                    .username(username)
                    .password(password)
                    .version(Version.V_1_0)
                    .build();
            session.open(false);
            session.setFetchSize(maxSize);
            //Set time zone
            session.setTimeZone(" + 08:00");
        }
        return session;
    }
 
    /**
     * Node path
     * @param records
     * @return
     */
    private List<String> getDeviceIds(List<? extends IoTDBRecordable> records) {<!-- -->
        List<String> deviceIds = new ArrayList<>();
        for (IoTDBRecordable ioTDBRecordable : records) {<!-- -->
            IoTDBRecord ioTDBRecord = ioTDBRecordable.toRecord();
            String deviceId = ioTDBRecord.getDeviceId();
            deviceIds.add(deviceId);
        }
        return deviceIds;
    }
 
    /**
     * timestamp
     * @param records
     * @return
     */
    private List<Long> getTimes(List<? extends IoTDBRecordable> records) {<!-- -->
        List<Long> times = new ArrayList<>();
        for (IoTDBRecordable ioTDBRecordable : records) {<!-- -->
            IoTDBRecord ioTDBRecord = ioTDBRecordable.toRecord();
            times.add(ioTDBRecord.getTime());
        }
        return times;
    }
 
    /**
     *Physical quantity, namely: attribute
     * @param records
     * @return
     */
    private List<List<String>> getMeasurementsList(List<? extends IoTDBRecordable> records) {<!-- -->
        List<List<String>> measurementsList = new ArrayList<>();
        for (IoTDBRecordable ioTDBRecordable : records) {<!-- -->
            IoTDBRecord ioTDBRecord = ioTDBRecordable.toRecord();
            measurementsList.add(ioTDBRecord.getMeasurementsList());
        }
        return measurementsList;
    }
 
    /**
     * Attribute value --- The attribute must correspond to the attribute value one-to-one
     * @param records
     * @return
     */
    private List<List<Object>> getValuesList(List<? extends IoTDBRecordable> records) {<!-- -->
        List<List<Object>> valuesList = new ArrayList<>();
        for (IoTDBRecordable ioTDBRecordable : records) {<!-- -->
            IoTDBRecord ioTDBRecord = ioTDBRecordable.toRecord();
            valuesList.add(ioTDBRecord.getValuesList());
        }
 
        return valuesList;
    }
 
    /**
     * Data type BOOLEAN((byte)0), INT32((byte)1),INT64((byte)2),FLOAT((byte)3),DOUBLE((byte)4),TEXT((byte)5) );
     * @param records
     * @return
     */
    private List<List<TSDataType>> getTSDataType(List<? extends IoTDBRecordable> records) {<!-- -->
        List<List<TSDataType>> valuesList = new ArrayList<>();
        for (IoTDBRecordable ioTDBRecordable : records) {<!-- -->
            IoTDBRecord ioTDBRecord = ioTDBRecordable.toRecord();
            List<TSDataType> strList = new ArrayList<>();
            for(String str : ioTDBRecord.getTypeList()){<!-- -->
                strList.add(convertTypeByEntity(str));
            }
            valuesList.add(strList);
        }
 
        return valuesList;
    }
 
    /**
     * Entity data type conversion
     * @param type attribute type
     * @return
     */
    private TSDataType convertTypeByEntity(String type) {<!-- -->
        switch (type) {<!-- -->
            case "java.lang.Double":
                return TSDataType.DOUBLE;
            case "java.lang.Integer":
                return TSDataType.INT32;
            case "java.lang.Long":
                return TSDataType.INT64;
            case "java.lang.Boolean":
                return TSDataType.BOOLEAN;
            case "java.lang.Float":
                return TSDataType.FLOAT;
            default:
                return TSDataType.TEXT;
        }
    }
 
    /**
     * Batch insert
     * @param records class collection
     */
    public void insertRecords(List<? extends IoTDBRecordable> records) {<!-- -->
        try {<!-- -->
            session.insertRecords(getDeviceIds(records), getTimes(records), getMeasurementsList(records),getTSDataType(records),
                    getValuesList(records));
        } catch (Exception e) {<!-- -->
            logger.error("IoTDB insertion exception:{}",e.getMessage());
        }
    }
 
    /**
     * Single inserted entity
     * @param recordEntity
     */
    public void insertRecord(IoTDBRecordable recordEntity) {<!-- -->
        try {<!-- -->
            IoTDBRecord ioTDBRecord = recordEntity.toRecord();
            List<TSDataType> strList = new ArrayList<>();
            for(String str : ioTDBRecord.getTypeList()){<!-- -->
                strList.add(convertTypeByEntity(str));
            }
            session.insertRecord(ioTDBRecord.getDeviceId(), ioTDBRecord.getTime(), ioTDBRecord.getMeasurementsList()
                    ,strList, ioTDBRecord.getValuesList());
        } catch (Exception e) {<!-- -->
            logger.error("IoTDB insertion exception:{}",e.getMessage());
        }
    }
 
    /**
     * description: Based on SQL query
     */
    public SessionDataSet query(String sql) throws StatementExecutionException, IoTDBConnectionException {<!-- -->
        return session.executeQueryStatement(sql,30000);
    }
 
    /**
     * description: Delete group such as root.a1eaKSRpRty
     * @param groupName: group name
     * @return
     */
    public void deleteStorageGroup(String groupName) throws StatementExecutionException, IoTDBConnectionException {<!-- -->
        session.deleteStorageGroup(groupName);
    }
 
    /**
     * description: Delete according to Timeseries such as: root.a1eaKSRpRty.CA3013A303A25467.breath (personal understanding: it is a specific physical quantity)
     */
    public void deleteTimeseries(String timeseries) throws StatementExecutionException, IoTDBConnectionException {<!-- -->
        session.deleteTimeseries(timeseries);
    }
    /**
     * description: Batch deletion based on Timeseries
     */
    public void deleteTimeserieList(List<String> timeseriesList) throws StatementExecutionException, IoTDBConnectionException {<!-- -->
        session.deleteTimeseries(timeseriesList);
    }
 
    /**
     * description: Batch deletion based on grouping
     */
    public void deleteStorageGroupList(List<String> storageGroupList) throws StatementExecutionException, IoTDBConnectionException {<!-- -->
        session.deleteStorageGroups(storageGroupList);
    }
 
    /**
     * description: Delete all data before the end time based on the path and end time
     */
    public void deleteDataByPathAndEndTime(String path, Long endTime) throws StatementExecutionException, IoTDBConnectionException {<!-- -->
        session.deleteData(path, endTime);
    }
    /**
     * description: Batch delete all data before the end time based on path collection and end time
     */
    public void deleteDataByPathListAndEndTime(List<String> pathList, Long endTime) throws StatementExecutionException, IoTDBConnectionException {<!-- -->
        session.deleteData(pathList, endTime);
    }
    /**
     * description: Batch deletion based on path collection and time period
     */
    public void deleteDataByPathListAndTime(List<String> pathList, Long startTime,Long endTime) throws StatementExecutionException, IoTDBConnectionException {<!-- -->
        session.deleteData(pathList, startTime, endTime);
    }
}
 
 
 
4. IoTDBRecord encapsulation

Mainly to physically encapsulate the IoTDB input parameters for ease of use, and

 
 
import java.util.List;
 
/**
 * IoTDB parameter entity class
 * @author zgy
 * @date 2023-6
 */
public class IoTDBRecord {<!-- -->
 
    /**
     * Node path
     */
    private String deviceId;
    /**
     * timestamp
     */
    private long time;
    /**
     * Attributes
     */
    private List<String> measurementsList;
    /**
     * attribute value
     */
    private List<Object> valuesList;
    /**
     * type of data
     */
    private List<String> typeList;
 
    public String getDeviceId() {<!-- -->
        return deviceId;
    }
 
    public void setDeviceId(String deviceId) {<!-- -->
        this.deviceId = deviceId;
    }
 
    public long getTime() {<!-- -->
        return time;
    }
 
    public void setTime(long time) {<!-- -->
        this.time = time;
    }
 
    public List<String> getMeasurementsList() {<!-- -->
        return measurementsList;
    }
 
    public void setMeasurementsList(List<String> measurementsList) {<!-- -->
        this.measurementsList = measurementsList;
    }
 
    public List<Object> getValuesList() {<!-- -->
        return valuesList;
    }
 
    public void setValuesList(List<Object> valuesList) {<!-- -->
        this.valuesList = valuesList;
    }
 
    public List<String> getTypeList() {<!-- -->
        return typeList;
    }
 
    public void setTypeList(List<String> typeList) {<!-- -->
        this.typeList = typeList;
    }
}
5. Custom entity listing annotations

 
import java.lang.annotation.*;
 
/**
 * Customize the current entity column name name
 * @author zgy
 * @date 2023-6
 */
@Documented
@Retention(RetentionPolicy.RUNTIME)
@Target({<!-- -->ElementType.TYPE, ElementType.ANNOTATION_TYPE})
public @interface IoTTableName {<!-- -->
 
    /**
     * Table name corresponding to the entity
     */
    String value() default "";
 
}
6. Interface acquisition information encapsulation

It mainly parses the entity class information into IoTDB input entity information, which is convenient for use and storage.

 
import cn.hutool.core.date.DateUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
 
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.List;
 
/**
 * @Author: ZGY
 * @Date: 2023/5/29 15:27
 * @FileName: Recordable
 * @Description: iot base class
 */
public interface IoTDBRecordable {<!-- -->
    Logger logger = LoggerFactory.getLogger(IoTDBRecordable.class);
    /**
     * Data loading method
     * @return Record
     */
    default IoTDBRecord toRecord() {<!-- -->
        IoTDBRecord ioTDBRecord = new IoTDBRecord();
        //current time
        ioTDBRecord.setTime(DateUtil.currentSeconds()*1000);
        Class aClass = this.getClass();
        IoTTableName name = this.getClass().getAnnotation(IoTTableName.class);
        ioTDBRecord.setDeviceId(name.value());
        Field[] declaredFields = aClass.getDeclaredFields();
        List<String> measurements = new ArrayList<>();
        List<Object> records = new ArrayList<>();
        List<String> types = new ArrayList<>();
        try {<!-- -->
            for (Field field : declaredFields) {<!-- -->
                measurements.add(field.getName());
                String methodNamePro = field.getName().substring(0, 1).toUpperCase() + field.getName().substring(1);
                Method methodName = this.getClass().getMethod("get" + methodNamePro);
                records.add(methodName.invoke(this));
                types.add(methodName.getReturnType().getName());
            }
            ioTDBRecord.setMeasurementsList(measurements);
            ioTDBRecord.setValuesList(records);
            ioTDBRecord.setTypeList(types);
        } catch (Exception e) {<!-- -->
            logger.error("IoTDB entity class conversion exception: {}",e.getMessage());
        }
        return ioTDBRecord;
    }
}
7. Node path parameter class

Unified definition for easy use, such as receiving syslog information

 
 
public class IoTDBTableParam {<!-- -->
 
 
    public static final String SYSLOG_IOT_TABLE = "root.syslog";
 
}
8. Examples of entity classes

Define the syslog entity class as an example, and introduce unified configuration parameter information through @IoTTableName for dynamic identification

 
 
import com.IoTDB.IoTDBRecordable;
import com.IoTDB.IoTDBTableParam;
import com.IoTDB.IoTTableName;
 
@IoTTableName(value = IoTDBTableParam.SYSLOG_IOT_TABLE)
public class IoTDBSysLog implements IoTDBRecordable {<!-- -->
    private String id;//primary key identification
    /**
     * facility Chinese logo
     */
    private String facilityName;
    /**
     * facility key
     */
    private Integer facilityKey;
    /**
     *Level Chinese logo
     */
    private String levelName;
    /**
     *Level Chinese identification key
     */
    private Integer levelKey;
    /**
     * Log IP
     */
    private String logIp;
    /**
     * Log messages
     */
    private String logMsg;
    private String createTime;
 
    public String getId() {<!-- -->
        return id;
    }
 
    public void setId(String id) {<!-- -->
        this.id = id;
    }
 
    public String getFacilityName() {<!-- -->
        return facilityName;
    }
 
    public void setFacilityName(String facilityName) {<!-- -->
        this.facilityName = facilityName;
    }
 
    public Integer getFacilityKey() {<!-- -->
        return facilityKey;
    }
 
    public void setFacilityKey(Integer facilityKey) {<!-- -->
        this.facilityKey = facilityKey;
    }
 
    public String getLevelName() {<!-- -->
        return levelName;
    }
 
    public void setLevelName(String levelName) {<!-- -->
        this.levelName = levelName;
    }
 
    public Integer getLevelKey() {<!-- -->
        return levelKey;
    }
 
    public void setLevelKey(Integer levelKey) {<!-- -->
        this.levelKey = levelKey;
    }
 
    public String getLogIp() {<!-- -->
        return logIp;
    }
 
    public void setLogIp(String logIp) {<!-- -->
        this.logIp = logIp;
    }
 
    public String getLogMsg() {<!-- -->
        return logMsg;
    }
 
    public void setLogMsg(String logMsg) {<!-- -->
        this.logMsg = logMsg;
    }
 
    public String getCreateTime() {<!-- -->
        return createTime;
    }
 
    public void setCreateTime(String createTime) {<!-- -->
        this.createTime = createTime;
    }
}
9. Insert data test

Let's write a test

    @Test
    public void testMongoDB(){<!-- -->
        //Insert into IoTDB history database
        IoTDBSysLog sysLog = new IoTDBSysLog();
        sysLog.setId("123");
        sysLog.setLogMsg("hello");
        sysLog.setLogIp("1.1.1.1");
        sysLog.setFacilityKey(1);
        sysLog.setFacilityName("information");
        sysLog.setLevelKey(3);
        sysLog.setLevelName("prompt information");
        sysLog.setCreateTime(DateUtil.now());
        iotDBSessionConfig.insertRecord(sysLog);
    }