When we upload files, if the file is too large, the request may time out. Therefore, when you need to upload a large file, you need to upload the file in parts. At the same time, if the file is too large and the network is not good, how can the file be resumed at a breakpoint? It is also necessary to record the currently uploaded file, and then make a judgment when the next upload request is made.
front end
- index.html
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta http-equiv="X-UA-Compatible" content="ie=edge">
<title>File upload</title>
<script src="//i2.wp.com/cdn.bootcss.com/axios/0.18.0/axios.min.js"></script>
<script src="//i2.wp.com/code.jquery.com/jquery-3.4.1.js"></script>
<script src="./spark-md5.min.js"></script>
<script>
$(document).ready(() => {
const chunkSize = 1 * 1024 * 1024; //The size of each chunk is set to 1 megabyte
// Use the Blob.slice method to split the file.
// At the same time, this method is used differently in different browsers.
const blobSlice =
File.prototype.slice || File.prototype.mozSlice || File.prototype.webkitSlice;
const hashFile = (file) => {
return new Promise((resolve, reject) => {
const chunks = Math.ceil(file.size / chunkSize);
let currentChunk = 0;
const spark = new SparkMD5.ArrayBuffer();
const fileReader = new FileReader();
function loadNext() {
const start = currentChunk * chunkSize;
const end = start + chunkSize >= file.size ? file.size : start + chunkSize;
fileReader.readAsArrayBuffer(blobSlice.call(file, start, end));
}
fileReader.onload = e => {
spark.append(e.target.result); // Append array buffer
currentChunk + = 1;
if (currentChunk < chunks) {
loadNext();
} else {
console.log('finished loading');
const result = spark.end();
// If you simply use result as the hash value, if the file contents are the same but the names are different
// If you want to keep two files, you cannot keep them. So add the file name.
const sparkMd5 = new SparkMD5();
sparkMd5.append(result);
sparkMd5.append(file.name);
const hexHash = sparkMd5.end();
resolve(hexHash);
}
};
fileReader.onerror = () => {
console.warn('File reading failed!');
};
loadNext();
}).catch(err => {
console.log(err);
});
}
const submitBtn = $('#submitBtn');
submitBtn.on('click', async () => {
const fileDom = $('#file')[0];
//The files obtained are an array of File objects. If multiple selections are allowed, there will be multiple files.
const files = fileDom.files;
const file = files[0];
if (!file) {
alert('No file obtained');
return;
}
const blockCount = Math.ceil(file.size / chunkSize); //Total number of fragments
const axiosPromiseArray = []; // axiosPromise array
const hash = await hashFile(file); //File hash
// After obtaining the file hash, if you need to resume the upload, you can verify it in the background based on the hash value.
// Check to see if the file has been uploaded, and if the transfer has been completed and the uploaded slices have been uploaded.
console.log(hash);
for (let i = 0; i < blockCount; i + + ) {
const start = i * chunkSize;
const end = Math.min(file.size, start + chunkSize);
// Build the form
const form = new FormData();
form.append('file', blobSlice.call(file, start, end));
form.append('name', file.name);
form.append('total', blockCount);
form.append('index', i);
form.append('size', file.size);
form.append('hash', hash);
// ajax submits fragments, at this time content-type is multipart/form-data
const axiosOptions = {
onUploadProgress: e => {
// Handle upload progress
console.log(blockCount, i, e, file);
},
};
//Add to Promise array
axiosPromiseArray.push(axios.post('/file/upload', form, axiosOptions));
}
//After all shards are uploaded, request to merge the shard files
await axios.all(axiosPromiseArray).then(() => {
// merge chunks
const data = {
size: file.size,
name: file.name,
total: blockCount,
hash
};
axios
.post('/file/merge_chunks', data)
.then(res => {
console.log('Upload successful');
console.log(res.data, file);
alert('Upload successful');
})
.catch(err => {
console.log(err);
});
});
});
})
window.onload = () => {
}
</script>
</head>
<body>
<h1>Large file upload test</h1>
<section>
<h3>Customized upload files</h3>
<input id="file" type="file" name="avatar"/>
<div>
<input id="submitBtn" type="button" value="Submit">
</div>
</section>
</body>
</html>
- dependent files
axios.js
jquery
spark-md5.js
rear end
- app.js
const Koa = require('koa');
const app = new Koa();
const Router = require('koa-router');
const multer = require('koa-multer');
const serve = require('koa-static');
const path = require('path');
const fs = require('fs-extra');
const koaBody = require('koa-body');
const { mkdirsSync } = require('./utils/dir');
const uploadPath = path.join(__dirname, 'uploads');
const uploadTempPath = path.join(uploadPath, 'temp');
const upload = multer({ dest: uploadTempPath });
const router = new Router();
app.use(koaBody());
/**
* single(fieldname)
* Accept a single file with the name fieldname. The single file will be stored in req.file.
*/
router.post('/file/upload', upload.single('file'), async (ctx, next) => {
console.log('file upload...')
//Create a folder based on the file hash, and move the default uploaded file to the current hash folder. Facilitate subsequent file merging.
const {
name,
total,
index,
size,
hash
} = ctx.req.body;
const chunksPath = path.join(uploadPath, hash, '/');
if(!fs.existsSync(chunksPath)) mkdirsSync(chunksPath);
fs.renameSync(ctx.req.file.path, chunksPath + hash + '-' + index);
ctx.status = 200;
ctx.res.end('Success');
})
router.post('/file/merge_chunks', async (ctx, next) => {
const {
size, name, total, hash
} = ctx.request.body;
//According to the hash value, obtain the fragmented file.
//Create storage file
// merge
const chunksPath = path.join(uploadPath, hash, '/');
const filePath = path.join(uploadPath, name);
// Read all chunks file names and store them in the array
const chunks = fs.readdirSync(chunksPath);
//Create storage file
fs.writeFileSync(filePath, '');
if(chunks.length !== total || chunks.length === 0) {
ctx.status = 200;
ctx.res.end('The number of slice files does not match');
return;
}
for (let i = 0; i < total; i + + ) {
//Append to the file
fs.appendFileSync(filePath, fs.readFileSync(chunksPath + hash + '-' + i));
//Delete the chunk used this time
fs.unlinkSync(chunksPath + hash + '-' + i);
}
fs.rmdirSync(chunksPath);
// The files are merged successfully and the file information can be stored in the database.
ctx.status = 200;
ctx.res.end('Merge successful');
})
app.use(router.routes());
app.use(router.allowedMethods());
app.use(serve(__dirname + '/static'));
app.listen(9000);
- utils/dir.js
const path = require('path');
const fs = require('fs-extra');
const mkdirsSync = (dirname) => {
if(fs.existsSync(dirname)) {
return true;
} else {
if (mkdirsSync(path.dirname(dirname))) {
fs.mkdirSync(dirname);
return true;
}
}
}
module.exports = {
mkdirsSync
};
Operation steps instructions
Server-side construction
Our following operations are guaranteed to be performed on the premise that node and npm have been installed. For the installation and use of node, please refer to the official website.
Create a new project folder file-upload
Initialize a project using npm: cd file-upload & amp; & amp; npm init
Install related dependencies
npm i koa npm i koa-router --save // Koa routing npm i koa-multer --save //File upload processing module npm i koa-static --save // Koa static resource processing module npm i fs-extra --save // file processing npm i koa-body --save //Request parameter parsing
Create project structure
file-upload
- static
-index.html
- spark-md5.min.js
- uploads
-temp
-utils
- dir.js
-app.js
Just copy the corresponding code to the specified location
Project startup: node app.js (you can use nodemon to manage services)
Visit: http://localhost:9000/index.html
Reference article: http://blog.ncmem.com/wordpress/2023/10/09/Segmented upload of large files based on node-js/
Welcome to join the group to discuss
