Due to the qrcode-decoder used in the actual application of the project, it was found that the recognition success rate is too low, and the scanned documents often cannot recognize the QR code, or the QR code with icon cannot be recognized. The rate is also terrible. Later, opencv-js-qrcode was introduced for further optimization and improvement, further improvingvue, js, html, and recognizing invoices with QR codes. The front-end supports converting pdf to images The QR code of this article Identification success rate.
The first thing I thought of was to intercept the QR code in the upper left corner of the invoice and then zoom in to improve the scanning success rate. Then I directly intercepted the QR code area and uploaded it for scanning. I found that the success rate was still too low, or it could not be recognized at all. So I planned to use Java background processing, and found that the success rate of using the QR code tool under the hutool package is the same as that of the front end. What the front end recognizes can also be recognized by the back end, and what it cannot recognize cannot be recognized…
But I can scan it out directly with my mobile phone, or I can scan it out directly by scanning on WeChat. I found that there may be a problem with the scanning method. I surfed the Internet for a while and found that JS recognized photos on Zhihu. Or the QR code in the picture is the most reliable article:
github:https://github.com/leidenglai/opencv-js-qrcode
opencv-js-qrcode demo link:https://leidenglai.github.io/opencv-js-qrcode/
After testing, I found that the function is indeed powerful, so I downloaded the resources and packaged them into the system:
opencvUtils.js
/**
* Scan object
* @constructor
*/
function OpencvUtils() {
this.loadScript = function (url) {
return new Promise((resolve, reject) => {
let script = document.createElement('script');
script.setAttribute('async', '');
script.setAttribute('type', 'text/javascript');
script.setAttribute('id', 'opencvjs');
script.addEventListener('load', async () => {
if (cv.getBuildInformation) {
resolve();
} else {
// WASM
if (cv instanceof Promise) {
cv = await cv;
resolve();
} else {
cv['onRuntimeInitialized'] = () => {
resolve();
};
}
}
});
script.addEventListener('error', () => {
reject();
});
script.src = url;
let node = document.getElementsByTagName('script')[0];
node.parentNode.insertBefore(script, node);
});
};
/**
* Request QR code training model file
* @param modelUrl
* @returns {Promise<Uint8Array>}
*/
this.fetchModelsData = async function (modelUrl) {
// Here are the files placed under public
const response = await fetch(modelUrl, {
method: 'GET'
});
const data = await response.arrayBuffer();
return new Uint8Array(data);
};
/**
* Load images into canvas
* The QR code of the invoice is basically in the upper left corner
* In order to improve efficiency, only the upper left corner area of the image QR code is intercepted and put into canvas
* @param {*} url
* @param {*} callBack
*/
this.loadImageToCanvas = function (url, callBack) {
let canvas = document.createElement('canvas');
canvas.id = 'openCVCanvasInput';
let ctx = canvas.getContext('2d');
let img = new Image();
img.crossOrigin = 'anonymous';
img.onload = () => {
const { width, height } = img;
const isVertical = width < height;
const crossNum = isVertical ? 3 : 4;
const verticalNum = isVertical ? 4 : 3;
canvas.width = width / crossNum;
canvas.height = height / verticalNum;
ctx.drawImage(img, isVertical ? width * (2 / 3) : 0, 0, width, height, 0, 0, width, height);
// Callback after loading
if (callBack) {
callBack();
}
};
document.body.appendChild(canvas);
img.src = url;
};
/**
* canvas to image
*/
this.imagedataToImage = function (imagedata) {
const canvas = document.createElement('canvas');
const ctx = canvas.getContext('2d');
canvas.width = imagedata.width;
canvas.height = imagedata.height;
ctx.putImageData(imagedata, 0, 0);
return new Promise(resolve => {
const img = new Image();
img.src = canvas.toDataURL();
img.onload = () => {
resolve(img);
};
});
};
/**
* Split image coordinates
* @param {*} width image width
* @param {*} height picture height
*
* @returns coordinate array [x,y,width,height][]
*/
this.segmentationImageCoordinates = function (width, height) {
const isVertical = width < height;
const crossNum = isVertical ? 3 : 5;
const verticalNum = isVertical ? 5 : 3;
const blockWidth = width / crossNum;
const blockHeight = height / verticalNum;
const coordinates = [];
for (let y = 0; y < verticalNum; y + + ) {
for (let x = 0; x < crossNum; x + + ) {
const cx = x * blockWidth;
const cy = y * blockHeight;
coordinates.push([cx, cy, blockWidth, blockHeight]);
}
}
return coordinates;
};
}
// Temporary scan object
let qrcode_detector = undefined;
//Temporary tool object
let tempUtils = null;
/**
* Load the model for initialization
* @returns {Promise<void>}
*/
async function loadModels() {
if (qrcode_detector !== undefined) {
console.log('Model Existed');
} else {
//Read the scan model under public
const dp = await tempUtils.fetchModelsData('/static/models/detect.prototxt');
const dw = await tempUtils.fetchModelsData('/static/models/detect.caffemodel');
const sp = await tempUtils.fetchModelsData('/static/models/sr.prototxt');
const sw = await tempUtils.fetchModelsData('/static/models/sr.caffemodel');
//Create a temporary folder for cv to store model data
cv.FS_createDataFile('/', 'detect.prototxt', dp, true, false, false);
cv.FS_createDataFile('/', 'detect.caffemodel', dw, true, false, false);
cv.FS_createDataFile('/', 'sr.prototxt', sp, true, false, false);
cv.FS_createDataFile('/', 'sr.caffemodel', sw, true, false, false);
//Create scan method
qrcode_detector = new cv.wechat_qrcode_WeChatQRCode('detect.prototxt', 'detect.caffemodel', 'sr.prototxt', 'sr.caffemodel');
console.log('OpenCV Model Created');
}
}
// initialization
function initOpencv() {
//Create temporary object
tempUtils = new OpencvUtils();
//Load opencv.js into the page
tempUtils
.loadScript('/static/opencv.js')
.then(() => {
//Load model
loadModels();
})
.catch(e => {
console.log('Failed to load ' + 'opencv.js', e);
});
}
// initialization
initOpencv();
/**
* opencv scan QR code
* @param qrcodeUrl
* @param callBack
*/
export function qrcode_run(qrcodeUrl, callBack) {
console.time('OpenCV takes time');
//Create canvas
tempUtils.loadImageToCanvas(qrcodeUrl, () => {
let inputImage = cv.imread('openCVCanvasInput', cv.IMREAD_GRAYSCALE);
let points_vec = new cv.MatVector();
let res = qrcode_detector.detectAndDecode(inputImage, points_vec);
if (res.size() > 0) {
console.log('opencv recognition result:', res.get(0));
if (callBack) {
callBack(res?.get(0) || null);
}
/* This step is to put the QR code taken from the screenshot into the canvas for display. I didn’t open it if I didn’t need it.
// document.querySelector('#qrcodeResult span').innerHTML = res.get(0);
let points = points_vec.get(0);
let x = points.floatAt(0);
let y = points.floatAt(1);
let width = points.floatAt(4) - points.floatAt(0);
let height = points.floatAt(5) - points.floatAt(1);
let rect = new cv.Rect(x, y, width, height);
dst = inputImage.roi(rect);
cv.imshow('qrcodeCanvasOutput', dst);*/
}
//Delete canvas object
document.getElementById('openCVCanvasInput').remove(); // Remove element after download is complete
console.timeEnd('OpenCV takes time');
});
}
Resource files are placed in the public directory to facilitate requests to access binary data:
This resource requires you to download the resource yourself from git: https://github.com/leidenglai/opencv-js-qrcode
Instructions:
import { qrcode_run } from '@/utils/opencv/opencvUtils';
After being introduced, the initialization method will be triggered directly. Because the resource to be loaded is almost 5M in size, it is recommended to directly introduce the load to prevent waiting when using it;
//Initialize opencv data initOpencv(); Life cycle process: 1. Load opencv.js into the page 2. After the loading is completed, call the model data loading 3. After the model file is successfully obtained, convert the resource file into a Uint8Array and load it into the engine. 4.Initialization completed 5. Call Scan 6. Create a canvas and load images into canvas 7. Recognize the image after it is loaded 8. Delete the canvas object after recognition
First call qrcode-decoder to scan (see the previous article for details). If no data can be scanned, call opencv for identification: 
Note: There are still some problems when using opencv to take screenshots. You can test it on the opencv-js-qrcode demo first, and then consider whether to use it in the project.
Like this, neither method can recognize it.