Part of the content of the remote xml
<imagelist name="FY4A AGRI IMG REGI MTCC GLL" tag="FY4A AGRI IMG REGI MTCC GLL"> <image time="2023-07-25 22:30 (UTC)" desc="FY4A AGRI IMG REGI MTCC GLL" url="http://img.nsmc.org.cn/PORTAL/FY4 /IMG/FY4A/AGRI/IMG/REGI/MTCC/GLL/FY4A-_AGRI--_N_REGI_1047E_L1C_MTCC_MULT_GLL_20230725223000_20230725223417_1000M_V0001.JPG"/> <image time="2023-07-25 22:23 (UTC)" desc="FY4A AGRI IMG REGI MTCC GLL" url="http://img.nsmc.org.cn/PORTAL/FY4 /IMG/FY4A/AGRI/IMG/REGI/MTCC/GLL/FY4A-_AGRI--_N_REGI_1047E_L1C_MTCC_MULT_GLL_20230725222336_20230725222753_1000M_V0001.JPG"/> </imagelist>
mq publisher timing task to send messages
@Component
public class TFYImage {<!-- -->
private ISyncFY syncFY;
Log log = LogFactory. getLog(TFYImage. class);
@Autowired
public TFYImage(ISyncFY syncFY){<!-- -->
this.syncFY = syncFY;
}
@PostConstruct
@Scheduled(cron = "0 0 * * * *")
public void Task1() {<!-- -->
log.info("execute FY Image task");
String xmlUrl = "xml path";
String localPath = "/media/resource/FY4APic/";
String type = "FY4A";
syncFY.syncFY4AImage(xmlUrl, localPath, type);
}
}
mq consumer
1. Remote xml reading
2. XML parsing, save the picture url in the image in the collection
3. Traversing the collection, when the picture does not exist locally, download the picture to the local
4. Pass the image path to the delay queue to delete the image later
5. Save custom image access path and other information to the database
import com.alibaba.fastjson.JSONObject;
import com.fasterxml.jackson.databind.JsonNode;
import com.sxqx.entity.MQMessage;
import com.sxqx.listener.IMessageReceiver;
import com.sxqx.mapper.remote.xugu1.fy.IFYImageMapper;
import com.sxqx.pojo.FYImageItem;
import com.sxqx.utils.common.ListUtils;
import com.sxqx.utils.dataConverter.JsonConverter;
import com.sxqx.utils.file.FileHelper;
import com.sxqx.utils.mq.MQMessageSender;
import com.sxqx.utils.xml.XMLUtil;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.amqp.rabbit.annotation.Queue;
import org.springframework.amqp.rabbit.annotation.RabbitHandler;
import org.springframework.amqp.rabbit.annotation.RabbitListener;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpression;
import javax.xml.xpath.XPathFactory;
import java.io.File;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.*;
@Component
public class SyncFYImgReceiver implements IMessageReceiver {<!-- -->
private final IFYImageMapper fyImageMapper;
private final MQMessageSender mqMessageSender;
@Autowired
public SyncFYImgReceiver(MQMessageSender mqMessageSender,
IFYImageMapper fyImageMapper
) {<!-- -->
this.mqMessageSender = mqMessageSender;
this.fyImageMapper = fyImageMapper;
}
Log log = LogFactory.getLog(SyncFYImgReceiver.class);
@RabbitListener(queuesToDeclare = {<!-- -->
@Queue(name = "sxqxgxw_sync_fy_img")
})
@RabbitHandler
@Override
public void onMessageReceived(String mqMessageString) {<!-- -->
JsonNode jsonNode = JsonConverter.jsonString2JsonNode(mqMessageString);
JsonNode msg = jsonNode. findValue("msg");
JsonNode JsonNodeParams = msg. findValue("params");
Map<String, Object> params = JsonConverter.jsonNode2HashMap(JsonNodeParams);
if (params. size() > 0) {<!-- -->
String xmlUrl = params. get("xmlUrl"). toString();
String localPath = params. get("localPath"). toString();
String type = params. get("type"). toString();
if(Objects.equals(type,"FY4A")){<!-- -->
List<String> imageUrlList = new ArrayList<>();
try {<!-- -->
//read xml
Document document = XMLUtil. readXMLUrl(xmlUrl);
// Create an XPath object
XPathFactory xPathFactory = XPathFactory. newInstance();
XPath xPath = xPathFactory. newXPath();
// Use XPath expression to get the imagelist node
XPathExpression expr = xPath.compile("/imagelist");
NodeList nodeList = (NodeList) expr.evaluate(document, XPathConstants.NODESET);
// traverse imagelist nodes
Node node = nodeList. item(0);
if (node.getNodeType() == Node.ELEMENT_NODE) {<!-- -->
Element imagelistElement = (Element) node;
// Get the name and tag attribute values
//String name = imagelistElement.getAttribute("name");
//String tag = imagelistElement.getAttribute("tag");
//System.out.println("name: " + name);
//System.out.println("tag: " + tag);
// Get image node list
NodeList imageNodes = imagelistElement.getElementsByTagName("image");
// traverse image nodes
//Read the image in xml and store it in imageUrlList
for (int j = 0; j < imageNodes. getLength(); j ++ ) {<!-- -->
Node imageNode = imageNodes. item(j);
if (imageNode.getNodeType() == Node.ELEMENT_NODE) {<!-- -->
Element imageElement = (Element) imageNode;
// Get the time, desc and url attribute values of the image node
//String time = imageElement.getAttribute("time");
//String desc = imageElement.getAttribute("desc");
String imageUrl = imageElement. getAttribute("url");
imageUrlList.add(imageUrl);
}
}
}
} catch (Exception e) {<!-- -->
e.printStackTrace();
}
List<FYImageItem> fyImageItems = new ArrayList<>();
//Download pictures, assemble data
for (String imageUrl : imageUrlList) {<!-- -->
// Get the position of the last slash
int slashIndex = imageUrl. lastIndexOf("/");
// intercept file name
String fileName = imageUrl. substring(slashIndex + 1);
String filePath = localPath;
String savePath = filePath + fileName;
File file = new File(savePath);
if (file. exists()) {<!-- -->
continue;
}
//download image
try {<!-- -->
long start = System. currentTimeMillis();
FileHelper.downloadUsingNIO(imageUrl, savePath);
long end = System. currentTimeMillis();
long timeDifferenceInSeconds = (end - start) / 1000;
log.info(savePath + "Download time:" + timeDifferenceInSeconds + "seconds");
} catch (IOException e) {<!-- -->
e.printStackTrace();
}
if (file.exists() & amp; & amp; file.length() > 0) {<!-- -->
//Assembly data
String[] s = fileName. split("_");
if (s. length == 13) {<!-- -->
try {<!-- -->
String product = s[0]. replace("-", "");
String dt = s[9].substring(0, 14);
SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss");
Date date = sdf. parse(dt);
Calendar bjc = Calendar. getInstance();
bjc.setTime(date);
bjc.add(Calendar.HOUR_OF_DAY, 8);
date = bjc. getTime();
FYImageItem fyImageItem = new FYImageItem();
fyImageItem.setTime(date);
fyImageItem.setPath("http://local server/mediaResource/FY4APic/" + fileName);
fyImageItem.setFilename(fileName);
fyImageItem.setProduct(product);
fyImageItems.add(fyImageItem);
} catch (Exception e) {<!-- -->
e.printStackTrace();
}
}
}
// clear data
MQMessage mqMessage = new MQMessage();
JSONObject jsonObject = new JSONObject();
jsonObject.put("filePath", savePath);
mqMessage.setMsg(jsonObject);
mqMessageSender.send("queue.file_delay_destroy", mqMessage);
}
//Save to database
if (fyImageItems. size() > 0) {<!-- -->
for (List<FYImageItem> fyImageItems1 : ListUtils. splitList(fyImageItems, 50)) {<!-- -->
int rows = fyImageMapper.insertIntoFYImage(fyImageItems1);
try {<!-- -->
Thread. sleep(50);
} catch (InterruptedException ex) {<!-- -->
ex. printStackTrace();
}
}
try {<!-- -->
Thread. sleep(100);
} catch (Exception e) {<!-- -->
}
} else {<!-- -->
log.info("don't need to sync fy4a file and fyImageItems.size() is " + fyImageItems.size());
}
}
}
}
}
XMLUtil reads remote xml links
import org.w3c.dom.Document;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import java.net.HttpURLConnection;
import java.net.URL;
public class XMLUtil {<!-- -->
public static Document readXMLUrl(String xmlUrl) throws Exception{<!-- -->
// Create a URL object
URL url = new URL(xmlUrl);
// Create a DocumentBuilderFactory object
DocumentBuilderFactory factory = DocumentBuilderFactory. newInstance();
// Create a DocumentBuilder object
DocumentBuilder builder = factory. newDocumentBuilder();
HttpURLConnection connection = (HttpURLConnection) url. openConnection();
// Set the connection timeout to 5 seconds
connection.setConnectTimeout(5000);
// Set the read timeout to 10 seconds
connection.setReadTimeout(10000);
// Read XML file from URL and parse
Document document = builder. parse(connection. getInputStream());
return document;
}
}
Download files using NIO
package com.sxqx.utils.file;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.net.URLConnection;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
public class FileHelper {<!-- -->
public static void deleteFile(File file) {<!-- -->
/*
File[] listFiles()
Returns an array of abstract pathnames denoting the files in the directory denoted by this abstract pathname.
*/
File[] files = file. listFiles();
if (files!=null) {<!-- -->//If the file is included, delete it
for (File value : files) {<!-- -->
if (value.isFile()) {<!-- -->
//delete child files
value. delete();
} else if (value.isDirectory()) {<!-- -->
// Find files in subdirectories recursively
deleteFile(value);
}
value.delete();//delete subdirectories
}
}
file.delete();
}
public static Boolean downloadFile(String urlString, String savePath) {<!-- -->
InputStream is = null;
FileOutputStream os = null;
try {<!-- -->
// Construct the URL
URL url = new URL(urlString);
// open the connection
URLConnection con = url. openConnection();
// input stream
is = con. getInputStream();
// 1K data buffer
byte[] bs = new byte[1024];
// read data length
int len;
// output file stream
File file = new File(savePath);
os = new FileOutputStream(file, true);
// start reading
while ((len = is.read(bs)) != -1) {<!-- -->
os.write(bs, 0, len);
}
return true;
} catch (Exception e) {<!-- -->
e.printStackTrace();
return false;
} finally {<!-- -->
// Done, close all connections
try {<!-- -->
if (null != os) {<!-- -->
os. close();
}
} catch (IOException e) {<!-- -->
e.printStackTrace();
}
try {<!-- -->
if (null != is) {<!-- -->
is. close();
}
} catch (IOException e) {<!-- -->
e.printStackTrace();
}
}
//When downloading a picture from WeChat, if there is no picture corresponding to the id, an empty picture will be downloaded, and there will be no return of null
}
/**
* Use NIO to download files
* @param urlStr
* @param file
* @throws IOException
*/
public static void downloadUsingNIO(String urlStr, String file) throws IOException {<!-- -->
URL url = new URL(urlStr);
ReadableByteChannel rbc = Channels. newChannel(url. openStream());
FileOutputStream fos = new FileOutputStream(file);
fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
fos. close();
rbc. close();
}
}
==================================================== =====================
Upload the attachment to the local path, and save the access path in the database
Controller layer code
Set access permissions for the method, configure the corresponding permissions in the database table, and only allow admin users to access
/**
* Data upload
* @param file uploaded attachment
* @param orderId application form id
* @return Result
*/
@PreAuthorize("hasAuthority('user:offline:uploadData')")
@PostMapping(value = "/uploadData")
public Result<Object> uploadData(MultipartFile file, String orderId) {<!-- -->
return applyOrderService.uploadData(file, orderId);
}
Configure the corresponding method access rights in the database directory table and role directory association table
service layer code
@Override
public Result<Object> uploadData(MultipartFile uploadFile, String orderId) {<!-- -->
//1, springsecurity uses the context to obtain user information
UsernamePasswordAuthenticationToken authentication = (UsernamePasswordAuthenticationToken) SecurityContextHolder.getContext().getAuthentication();
JWTUser jwtUser = (JWTUser) authentication. getPrincipal();
InputStream inputStream = null;
FileOutputStream fileOutputStream = null;
//2, query whether the order exists
LambdaQueryWrapper<ApplyOrderDO> queryWrapper = new LambdaQueryWrapper<>();
queryWrapper.eq(ApplyOrderDO::getOrderId, orderId);
ApplyOrderDO applyOrderDO = applyOrderMapper. selectOne(queryWrapper);
if (applyOrderDO == null) return Result.error(new CodeMsg(400, "The application form does not exist"));
if (applyOrderDO.getOrderStatus() == OrderConstant.APPLIED) {<!-- -->
return Result.error(new CodeMsg(400, "The application form has been submitted but not approved, please approve it before uploading the attachment"));
} else if (applyOrderDO.getOrderStatus() == OrderConstant.APPLY_REJECTED) {<!-- -->
return Result.error(new CodeMsg(400, "The application has been rejected"));
} else if (applyOrderDO.getOrderStatus() == OrderConstant.APPLY_FINISHED) {<!-- -->
return Result.error(new CodeMsg(400, "This application process has ended"));
}
//3, io stream upload attachment
String filename = "";
try {<!-- -->
filename = uploadFile. getOriginalFilename();
System.out.println(filename);
File dir = new File("/data/static/dataSharingStatic/offline/orderFile/" + orderId + "/");
if (!dir. exists()) {<!-- -->
boolean mkdirs = dir.mkdirs();
}
inputStream = uploadFile. getInputStream();
fileOutputStream = new FileOutputStream("/data/static/dataSharingStatic/offline/orderFile/" + orderId + "/" + filename);
IOUtils.copy(inputStream, fileOutputStream);
fileOutputStream. flush();
} catch (IOException ex) {<!-- -->
ex. printStackTrace();
} finally {<!-- -->
try {<!-- -->
if (inputStream != null) {<!-- -->
inputStream. close();
}
if (fileOutputStream != null) {<!-- -->
fileOutputStream. close();
}
} catch (IOException ex) {<!-- -->
ex. printStackTrace();
}
}
//4, save the attachment url after configuring nginx
applyOrderDO.setProcessAppendixUrl("http://localIp/dataSharingStatic/offline/orderFile/" + orderId + "/" + filename);
applyOrderDO.setUpdateTime(new Date());
applyOrderDO.setUpdateUserId(jwtUser.getUserDO().getUuid());
applyOrderDO.setUpdateDealMobile(jwtUser.getUserDO().getMobile());
applyOrderDO.setRemarks("The data has been uploaded, the application is completed");
int update = applyOrderMapper. update(applyOrderDO, queryWrapper);
if (update > 0) {<!-- -->
return Result.success("upload attachment successfully");
}
return Result.error(new CodeMsg(500, "Failed to upload attachment"));
}
postman trigger upload attachment request
