Look at the logic first
How to slice? How to get the unique hash of a file? Interact with the backend to obtain the status of file upload, which can be used to judge the situation. Is it a second upload or a resume upload? Upload a sliced file and re-execute the failed file? Notify the backend of all uploads completed?
1. Upload all the code first, then analyze it in part 2 and use it in vue in part 3.
Related dependencies
spark-md5 is mainly used to get the md5 of the file
mitt publish and subscribe
import SparkMD5 from 'spark-md5'
import {<!-- --> getStartMultipartUpload, postUploadMultipartPart, postCompleteMultipartUpload } from '@/api/file'
import mitt from '@/utils/mitt'
class UpLoadVideo {<!-- -->
/**
* file stream file
* sectionSize slice size (MB) default 5MB
* failCount number of failed retries, default 3 times
* concurrencyMax maximum concurrency
* @param {*} params
*/
constructor(params) {<!-- -->
// file stream
this.file = params.file
this.sectionSize = params.sectionSize || 5
this.failCount = params.failCount || 3
this.concurrencyMax = params.concurrencyMax || 1
// Whether to stop uploading
this.isStop = false
this._count = 0
// Each file will return the upload ID before checking the upload interface. If the md5 is the same, the returned ID will be the same.
this.uploadId = ''
// md5 case
this.md5 = {<!-- -->
value: '',
Progress: 0
}
//Upload progress
this.upProgress = 0
// 0 Not started 1 Slicing in progress 2 Slicing completed 3 Uploading starting 4 All uploads completed 5 Upload failed Status defined by the front end
this.status = 0
// Publish and subscribe
this.mitt = mitt
// slice list
this.fileList = []
//The initial number of slices
this.multipartCount = 0
//Video duration (milliseconds) is obtained locally
this.duration = 0
this.upVideo()
}
/**
* Get file md5 and slices
* And check the status of md5 whether it has been uploaded or is half uploaded
* @returns
*/
async upVideo() {<!-- -->
const file = this.file
this.mitt.emit('currentFunc', {<!-- --> msg: 'Checking file information' })
const md5 = await this.getMd5(2)
const duration = await this.getDuration(file)
this.duration = duration
const size = 1024 * 1024 * this.sectionSize // Slice size
const fileList = []
let index = 0 // slice sequence number
for (let cur = 0; cur < file.size; cur + = size) {<!-- -->
const sectionFile = file.slice(cur, cur + size)
const partIdx = + + index
fileList.push({<!-- -->
partIdx,
multipartName: file.name + '_' + partIdx,
file: sectionFile,
size: sectionFile.size
})
}
this.multipartCount = index
const res = await this.getDetail({<!-- -->
MD5: md5,
Size: file.size,
FileName: file.name,
MultipartCount: fileList.length
})
// slice array
const newList = []
// According to the data returned by the interface, partIdxList indicates those indexes that have been uploaded and filter out those that have not been uploaded.
if (res.partIdxList & amp; & amp; res.partIdxList.length) {<!-- -->
fileList.forEach(item => {<!-- -->
if (!res.partIdxList.includes(item.partIdx)) {<!-- -->
newList.push(item)
}
})
}
this.uploadId = res.uploadId
// state 2 uploaded
if (res.state === 2) {<!-- -->
this.status = 4
this.upEmit('currentFunc', {<!-- --> msg: 'Upload completed', ...res })
return
}
// state 1 is uploading and the slice length is equal to the length of the uploaded sequence number list
if (res.state === 1 & amp; & amp; fileList.length === res.partIdxList.length) {<!-- -->
this.getCompleteMultipartUpload(false)
return
}
this.fileList = newList.length ? newList : fileList
this.upSection()
}
//Query before uploading
async getDetail(data) {<!-- -->
return new Promise((resolve, reject) => {<!-- -->
getStartMultipartUpload(data).then(res => {<!-- -->
resolve(res.data)
})
})
}
/**
* Whether to check
* @param {*} is
*/
async getCompleteMultipartUpload(is = false) {<!-- -->
const formData = new FormData()
formData.append('UploadId', this.uploadId)
postCompleteMultipartUpload(formData).then(res => {<!-- -->
this.upProgress = 100
this.status = 4
this.upEmit('currentFunc', {<!-- --> msg: 'Upload completed', ...res.data })
})
}
async upSection(fileList) {<!-- -->
if (this._count === this.failCount) {<!-- -->
this.upProgress = 0
this.mitt.emit('currentFunc', {<!-- --> msg: 'The upload failed, please try uploading again. Your upload progress will be retained, and the next upload will be accelerated\ ' })
return
}
fileList = fileList || this.fileList
console.time()
if (fileList.length === 0) {<!-- -->
console.timeEnd()
this.getCompleteMultipartUpload(true)
return
}
const pool = []// concurrent pool
const max = this.concurrencyMax // Maximum concurrency
let finish = 0//Quantity completed
const failList = []//Failed list
const upProgress = this.upProgress
if (!upProgress) {<!-- -->
// Delay execution for 1 second and the number completed is greater than 0
setTimeout(() => {<!-- -->
if (!finish) {<!-- -->
this.mitt.emit('currentFunc', {<!-- --> msg: 'Ready to upload...' })
}
}, 1000)
}
this.status = 3
for (let i = 0; i < fileList.length; i + + ) {<!-- -->
const item = fileList[i]
//Call the interface and upload slices
const task = this.apiFun(item).then(res => {<!-- -->
const progress = (100 / fileList.length * finish)
this.upProgress = Math.floor(progress * 100) / 100
this.upEmit('currentFunc', {<!-- --> msg: 'Uploading files' + this.upProgress + '%' })
// After the request ends, remove the Promise task from the concurrent pool
const index = pool.findIndex(t => t === task)
pool.splice(index)
}).catch(_ => {<!-- -->
//Failed ones are stored in the failure array
failList.push(item)
}).finally(_ => {<!-- -->
finish++
// After all requests are completed, check the failed array
if (finish === fileList.length) {<!-- -->
// Check the number of failures
this._count++
this.upSection(failList)
}
})
pool.push(task)
if (pool.length === max) {<!-- -->
//End upload
if (this.isStop) break
// Whenever the concurrency pool finishes running a task, another task is inserted
await Promise.allSettled(pool)
}
}
}
// Slice request upload
apiFun(item) {<!-- -->
return new Promise((resolve, reject) => {<!-- -->
const formData = new FormData()
formData.append('UploadId', this.uploadId)
formData.append('MultipartName', item.multipartName)
formData.append('PartIdx', item.partIdx)
formData.append('Size', item.size)
formData.append('File', item.file)
postUploadMultipartPart(formData).then(res => {<!-- -->
resolve(item.partIdx)
}).catch(err => {<!-- -->
reject(err)
})
})
}
upEmit(event = 'currentFunc', data) {<!-- -->
this.mitt.emit(event, {<!-- --> ...data, md5: this.md5, upProgress: this.upProgress, status: this.status, duration: this.duration })
}
/**
*
* @param {*} computeCount number of calculations, if not passed, the default is to calculate all slices
* @returns
*/
getMd5(computeCount) {<!-- -->
const that = this
return new Promise((resolve, reject) => {<!-- -->
// compatible
const blobSlice = File.prototype.slice || File.prototype.mozSlice || File.prototype.webkitSlice
//Slice calculation size
const chunkSize = 2097152 // Read in chunks of 2MB
// current file stream
const file = that.file
//How many slices can be divided currently?
let chunks = Math.ceil(file.size / chunkSize)
// If the number of times passed in is less than the number of slices, use the number passed in
if (computeCount & amp; & amp; computeCount < chunks) {<!-- -->
chunks = computeCount
}
let currentChunk = 0
//Library method for calculating md5
const spark = new SparkMD5.ArrayBuffer()
//Create FileReader
const fileReader = new FileReader()
//Write to slice stream
function loadNext() {<!-- -->
const start = currentChunk * chunkSize
const end = ((start + chunkSize) >= file.size) ? file.size : start + chunkSize
fileReader.readAsArrayBuffer(blobSlice.call(file, start, end))
}
loadNext()
// Calculate md5 after writing is completed
fileReader.onload = function(e) {<!-- -->
spark.append(e.target.result) //Append array buffer
currentChunk + +
if (currentChunk < chunks) {<!-- -->
const progress = ((100 / chunks) * currentChunk)
this.status = 1
that.md5 = {<!-- -->
value: '',
progress: Math.floor(progress * 100) / 100
}
that.upEmit('currentFunc', {<!-- --> msg: 'Checking file information' + that.md5.progress + '%' })
loadNext()
} else {<!-- -->
const md5 = {<!-- -->
value: spark.end(),
Progress: 100
}
that.md5 = md5
this.status = 2
that.upEmit('currentFunc', {<!-- --> msg: 'Check file information 100%' })
resolve(md5.value)
}
}
fileReader.onerror = function() {<!-- -->
that.upEmit('currentFunc', {<!-- --> msg: 'Failed to check file information, please try again' })
}
})
}
/**
* Get video length
* @param {*} file
* @returns
*/
getDuration(file) {<!-- -->
return new Promise((resolve, reject) => {<!-- -->
const video = document.createElement('video')
video.preload = 'metadata'
video.src = URL.createObjectURL(file)
video.onloadedmetadata = function() {<!-- -->
window.URL.revokeObjectURL(video.src)
const duration = video.duration
resolve(Math.floor(duration * 1000))
}
})
}
/**
*Stop uploading
*/
stopUpLoad() {<!-- -->
this.upProgress = 0
this.status = 5
this.isStop = true
this.mitt.emit('currentFunc', {<!-- --> msg: 'Upload failed, please try again' })
this.destroy()
}
destroy() {<!-- -->
this.file = null
this.mitt.clear()
}
}
export default UpLoadVideo
2 Code analysis (function)
The entire upload was done using Alioss.
The backend encapsulates 3 APIs. The first is query before uploading, the second is to upload video fragments, and the third is to complete the upload.
import {<!-- --> getStartMultipartUpload, postUploadMultipartPart, postCompleteMultipartUpload } from '@/api/file'
2.1 getMd5 is used to get the md5 of the file. I only take the first few slices as the global md5, because in the case of large files, getting the md5 of the file is also a time-consuming operation.
2.2 getDuration gets the video duration
2.3 UpVideo processing of the size of each slice and query before uploading, mainly processing whether it has been uploaded
2.3.1 Processing of slice size
const size = 1024 * 1024 * this.sectionSize // Slice size
const fileList = []
let index = 0 // slice sequence number
for (let cur = 0; cur < file.size; cur + = size) {<!-- -->
const sectionFile = file.slice(cur, cur + size)
const partIdx = + + index
fileList.push({<!-- -->
partIdx, // Used later to determine which slice has been uploaded
multipartName: file.name + '_' + partIdx,
file: sectionFile,
size: sectionFile.size
})
}
2.3.2 Query before uploading
const res = await this.getDetail({<!-- -->
MD5: md5,
Size: file.size,
FileName: file.name,
MultipartCount: fileList.length
})
// slice array
const newList = []
// According to the data returned by the interface, partIdxList indicates those indexes that have been uploaded and filter out those that have not been uploaded.
if (res.partIdxList & amp; & amp; res.partIdxList.length) {<!-- -->
fileList.forEach(item => {<!-- -->
if (!res.partIdxList.includes(item.partIdx)) {<!-- -->
newList.push(item)
}
})
}
this.uploadId = res.uploadId
// state 2 has been uploaded and returns the result directly
if (res.state === 2) {<!-- -->
this.status = 4
this.upEmit('currentFunc', {<!-- --> msg: 'Upload completed', ...res })
return
}
// state 1 is uploading and the slice length is equal to the length of the uploaded sequence number list. Directly call the upload completion interface.
if (res.state === 1 & amp; & amp; fileList.length === res.partIdxList.length) {<!-- -->
this.getCompleteMultipartUpload(false)
return
}
// Start upload
this.fileList = newList.length ? newList : fileList
this.upSection()
2.4 upSection starts multipart upload
if (this._count === this.failCount) {<!-- -->
this.upProgress = 0
this.mitt.emit('currentFunc', {<!-- --> msg: 'The upload failed, please try uploading again. Your upload progress will be retained, and the next upload will be accelerated\ ' })
return
}
fileList = fileList || this.fileList
console.time()
// All uploads are completed and the completion interface is called
if (fileList.length === 0) {<!-- -->
console.timeEnd()
this.getCompleteMultipartUpload(true)
return
}
const pool = []// concurrent pool
const max = this.concurrencyMax // Maximum concurrency
let finish = 0//Quantity completed
const failList = []//Failed list
const upProgress = this.upProgress
if (!upProgress) {<!-- -->
// Delay execution for 1 second and the number completed is greater than 0
setTimeout(() => {<!-- -->
if (!finish) {<!-- -->
this.mitt.emit('currentFunc', {<!-- --> msg: 'Ready to upload...' })
}
}, 1000)
}
this.status = 3
for (let i = 0; i < fileList.length; i + + ) {<!-- -->
const item = fileList[i]
//Call the interface and upload slices
const task = this.apiFun(item).then(res => {<!-- -->
const progress = (100 / fileList.length * finish)
this.upProgress = Math.floor(progress * 100) / 100
this.upEmit('currentFunc', {<!-- --> msg: 'Uploading files' + this.upProgress + '%' })
// After the request ends, remove the Promise task from the concurrent pool
const index = pool.findIndex(t => t === task)
pool.splice(index)
}).catch(_ => {<!-- -->
//Failed ones are stored in the failure array. Each time fileList is executed, the array of failList will be executed.
failList.push(item)
}).finally(_ => {<!-- -->
finish++
// After all requests are completed, check the failed array
if (finish === fileList.length) {<!-- -->
// Check the number of failures
this._count++
this.upSection(failList)
}
})
pool.push(task)
if (pool.length === max) {<!-- -->
//End upload
if (this.isStop) break
// Whenever the concurrency pool finishes running a task, another task is inserted
await Promise.allSettled(pool)
}
}
3. Use encapsulated js in vue
Imported into vue file
import UpLoadVideo from '@/utils/uploadVideo'
data(){<!-- -->
return {<!-- -->
uploadVideo: null
}
}
// fileBase is a stream file
let newUploadVideo = new UpLoadVideo({<!-- -->
file: fileBase
})
newUploadVideo.mitt.on('currentFunc', data => {<!-- -->
this.loadingText = data.msg // Some uploading tips
this.upProgress = data.upProgress // Upload progress
// status is customized state is returned from the background
if (data.status === 4 || data.state === 2) {<!-- -->
console.log('%c [The data in data is returned by the interface or returned in the encapsulated js]', 'font-size:13px; background:pink; color:#bf2c9f;', data)
this.upProgress = 0
this.loadingText = ''
newUploadVideo.destroy()
newUploadVideo = null
this.uploadVideo = null
}
})
this.uploadVideo = newUploadVideo
Reference article: http://blog.ncmem.com/wordpress/2023/11/01/Front-end large file slicing upload, breakpoint resume upload, second transfer and other solutions/
Welcome to join the group to discuss
