Hardware:
- RK3399 development board x1
- unoR3 development board x1
- UVC camera x1
- TB6612fng motor drive circuit x1
- DC worm motor x2
Third-party library
- OpenCV3.8.0 for Android
- CH34XUARTDriver
Development tools and environment
- Android Studio 4.1.2
- Arduino 1.8.9
- JDK 1.8
- NDK 21.1.6352462
unoR3 pin definition is as follows
#define PWMA 5 //Duty cycle pin of Motor1 #define PWMB 6 //Duty cycle pin of Motor2 #define STBY 10 //STBY pin #define AIN1 4 //IN1 pin of Motor1 #define AIN2 7 //IN2 pin of Motor1 #define BIN1 8 //IN1 pin of Motor2 #define BIN2 9 //IN2 pin of Motor2 //Motor 1 operates the head to move left and right //Motor 2 operates the head to move up and down
OpenCV import
It’s too troublesome. I don’t want to take screenshots. Please refer to this guy’s article.
There is a pit here, I don’t know if others have encountered it. There are spaces in my Android project directory, causing the import of OpenCV to always fail.
Import of CH34XUARTDriver
This is simple. After downloading, copy the CH34XUARTDriver.jar file in the lib directory to the project libs directory, and then right-click Add As Library.
Main code
unoR3 Arduino
#define PWMA 5 //Duty cycle pin of Motor1
#define PWMB 6 //Duty cycle pin of Motor2
#define STBY 10 //STBY pin, this pin is always high
#define AIN1 4 //IN1 pin of Motor1
#define AIN2 7 //IN2 pin of Motor1
#define BIN1 8 //IN1 pin of Motor2
#define BIN2 9 //IN2 pin of Motor2
boolean str = false;
String IOFlag = "";
void setup() {<!-- -->
Serial.begin(115200);
IOFlag.reserve(200);
Serial.println("Devices Init Successfully!!");
pinMode(4, OUTPUT);
pinMode(5, OUTPUT);
pinMode(6, OUTPUT);
pinMode(7, OUTPUT);
pinMode(8, OUTPUT);
pinMode(9, OUTPUT);
pinMode(10, OUTPUT);
}
void loop() {<!-- -->
if (str) {<!-- -->
int len = IOFlag.length();
checkMotor(IOFlag);
IOFlag = "";
str = false;
}
}
void startMotor() {<!-- -->
digitalWrite(STBY, HIGH);
}
void stopMotor() {<!-- -->
digitalWrite(STBY, LOW);
}
void z_motor1() {<!-- -->
startMotor();
digitalWrite(AIN2, LOW);
digitalWrite(AIN1, HIGH);
analogWrite(PWMA, 200);
}
void z_motor2() {<!-- -->
startMotor();
digitalWrite(BIN2, LOW);
digitalWrite(BIN1, HIGH);
analogWrite(PWMB, 200);
}
void f_motor1() {<!-- -->
startMotor();
digitalWrite(AIN1, LOW);
digitalWrite(AIN2, HIGH);
analogWrite(PWMA, 200);
}
void f_motor2() {<!-- -->
startMotor();
digitalWrite(BIN1, LOW);
digitalWrite(BIN2, HIGH);
analogWrite(PWMB, 200);
}
void checkMotor(String IOSring) {<!-- -->
if (IOString == "0100")
{<!-- -->
Serial.println("0100");
z_motor1();
}
if (IOString == "0101")
{<!-- -->
Serial.println("0101");
f_motor1();
}
if (IOString == "1000")
{<!-- -->
Serial.println("1000");
z_motor2();
}
if (IOString == "1001")
{<!-- -->
Serial.println("1001");
f_motor2();
}
if (IOString == "stopMotor") {<!-- -->
stopMotor();
}
}
void serialEvent() {<!-- -->
while (Serial.available()) {<!-- -->
char inChar = (char)Serial.read();
if (inChar != '\
')
IOFlag + = inChar;
if (inChar == '\
') {<!-- -->
str = true;
}
}
}
Android Studio
MainActivity
import androidx.appcompat.app.AppCompatActivity;
import androidx.core.app.ActivityCompat;
import androidx.core.content.ContextCompat;
import android.Manifest;
import android.content.Context;
import android.content.pm.PackageManager;
import android.os.Bundle;
import android.util.Log;
import android.view.SurfaceView;
import android.view.WindowManager;
import com.xuye.openeye.Head.Compensate;
import org.jetbrains.annotations.NotNull;
import org.opencv.android.BaseLoaderCallback;
import org.opencv.android.CameraBridgeViewBase;
import org.opencv.android.LoaderCallbackInterface;
import org.opencv.android.OpenCVLoader;
import org.opencv.core.Core;
import org.opencv.core.CvType;
import org.opencv.core.Mat;
import org.opencv.core.MatOfRect;
import org.opencv.core.Point;
import org.opencv.core.Rect;
import org.opencv.core.Scalar;
import org.opencv.core.Size;
import org.opencv.imgproc.Imgproc;
import org.opencv.objdetect.CascadeClassifier;
import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.util.HashMap;
import java.util.Map;
public class MainActivity extends AppCompatActivity implements CameraBridgeViewBase.CvCameraViewListener2 {<!-- -->
// Used to load the 'native-lib' library on application startup.
static {<!-- -->
System.loadLibrary("native-lib");
}
/**
* A native method that is implemented by the 'native-lib' native library,
* which is packaged with this application.
*/
//public native String stringFromJNI();
private CameraBridgeViewBase mCVCamera;
private CascadeClassifier cascadeClassifier; // OpenCV face detector
private Mat grayscaleImage;
private int absoluteFaceSize;
private static final String TAG = "OCV/BaseLoaderCallback";
private final int REQUEST_CAMERA_PERMISSION = 0x13;
/**
* Manage Android services through OpenCV and initialize OpenCV asynchronously
*/
private final BaseLoaderCallback mLoaderCallback = new BaseLoaderCallback(this) {<!-- -->
@Override
public void onManagerConnected(int status) {<!-- -->
if (status == LoaderCallbackInterface.SUCCESS) {<!-- -->
Log.i(TAG, "OpenCV loaded successfully");
mCVCamera.enableView();
} else {<!-- -->
super.onManagerConnected(status);
}
}
};
@Override
protected void onCreate(Bundle savedInstanceState) {<!-- -->
super.onCreate(savedInstanceState);
getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
setContentView(R.layout.activity_main);
if (ContextCompat.checkSelfPermission(this, Manifest.permission.CAMERA) != PackageManager.PERMISSION_GRANTED) {<!-- -->
ActivityCompat.requestPermissions(this, new String[]{<!-- -->Manifest.permission.CAMERA}, REQUEST_CAMERA_PERMISSION);
}
mCVCamera = findViewById(R.id.camera_view);
mCVCamera.setCameraPermissionGranted();
// mCVCamera.setCameraIndex(1);
mCVCamera.enableFpsMeter();
mCVCamera.setVisibility(SurfaceView.VISIBLE);
mCVCamera.setCvCameraViewListener(this);
Log.d(TAG, "onCreate: " + OpenCVLoader.OPENCV_VERSION);
}
@Override
public void onRequestPermissionsResult(int requestCode, @NotNull String[] permissions, @NotNull int[] grantResults) {<!-- -->
if (requestCode == REQUEST_CAMERA_PERMISSION) {<!-- -->
if (grantResults.length > 0 & amp; & amp; grantResults[0] == PackageManager.PERMISSION_GRANTED) {<!-- -->
// The camera permission has been authorized and camera-related operations can be performed.
mCVCamera.setCameraPermissionGranted();
Log.d(TAG, "onRequestPermissionsResult: The camera permission has been authorized and camera-related operations can be performed");
} else {<!-- -->
// Camera permission is denied and camera related operations cannot be performed.
Log.d(TAG, "onRequestPermissionsResult: Camera permission is denied, camera related operations cannot be performed");
}
}
}
@Override
public void onResume() {<!-- -->
super.onResume();
boolean flag = OpenCVLoader.initDebug();
if (!flag) {<!-- -->
Log.d(TAG, "OpenCV library not found!");
} else {<!-- -->
Log.d(TAG, "OpenCV library found inside package. Using it!");
initializeOpenCVDependencies();
mLoaderCallback.onManagerConnected(LoaderCallbackInterface.SUCCESS);
}
}
@Override
public void onCameraViewStarted(int width, int height) {<!-- -->
grayscaleImage = new Mat(height, width, CvType.CV_8UC4);
absoluteFaceSize = (int) (height * 0.2);
}
@Override
public void onCameraViewStopped() {<!-- -->
}
@Override
public Mat onCameraFrame(CameraBridgeViewBase.CvCameraViewFrame inputFrame) {<!-- -->
Mat aInputFrame = inputFrame.rgba();
Core.flip(aInputFrame, aInputFrame, 1);
Imgproc.cvtColor(aInputFrame, grayscaleImage, Imgproc.COLOR_RGBA2RGB);
MatOfRect faces = new MatOfRect();
if (cascadeClassifier != null) {<!-- -->
cascadeClassifier.detectMultiScale(grayscaleImage, faces, 1.1, 3, 2,
new Size(absoluteFaceSize, absoluteFaceSize), new Size());
}
Rect[] facesArray = faces.toArray();
Rect mRectFace = new Rect();
for (Rect rect : facesArray) {<!-- -->
//Calculate the difference in x coordinates between the two points of the face recognized this time
double f = rect.br().x - rect.tl().x;
//Check whether the face temporary storage object is empty
//The temporary object is empty
//Calculate the difference in x coordinates between two points of the temporary object's face
double _f = mRectFace.br().x - mRectFace.tl().x;
if (_f < f) {<!-- -->
//If the coordinate difference of the temporary object is smaller than the face coordinate difference detected in this loop, it means that the face recognized in this loop is larger than the face size in the temporary object;
//Assign the face obtained in this loop to the temporary object
mRectFace = rect;
}
}
Imgproc.rectangle(aInputFrame, mRectFace.tl(), mRectFace.br(), new Scalar(0, 255, 0, 255), 3);
// If no face is detected, return the MAT object directly
if (mRectFace.tl().x == 0 & amp; & amp; mRectFace.tl().y == 0 & amp; & amp; mRectFace.br().x == 0 & amp; & amp; mRectFace.br().y == 0)
return aInputFrame;
int width = aInputFrame.width();
int height = aInputFrame.height();
Point br = mRectFace.br();
Point tl = mRectFace.tl();
Log.d(TAG, "onCameraFrame: facesArray x:" + mRectFace.x + ".facesArray y" + mRectFace.y);
Map<String, Object> map = new HashMap<>();
map.put("width", width);
map.put("height", height);
map.put("br", br);
map.put("tl", tl);
Compensate mCompensate = new Compensate(map, getApplication(), this);
mCompensate.start();
return aInputFrame;
}
/*
**Initialize OPENCV face detection model
*/
private void initializeOpenCVDependencies() {<!-- -->
try {<!-- -->
InputStream is = getResources().openRawResource(R.raw.lbpcascade_frontalface);
File cascadeDir = getDir("cascade", Context.MODE_PRIVATE);
File mCascadeFile = new File(cascadeDir, "lbpcascade_frontalface.xml");
FileOutputStream os = new FileOutputStream(mCascadeFile);
byte[] buffer = new byte[4096];
int bytesRead;
while ((bytesRead = is.read(buffer)) != -1) {<!-- -->
os.write(buffer, 0, bytesRead);
}
is.close();
os.close();
cascadeClassifier = new CascadeClassifier(mCascadeFile.getAbsolutePath());
} catch (Exception e) {<!-- -->
Log.e("OpenCVActivity", "Error loading cascade", e);
}
mCVCamera.enableView();
}
}
Head position compensation class
import android.app.Application;
import android.content.Context;
import android.util.Log;
import com.xuye.openeye.Serial.Ch34;
import org.opencv.core.Point;
import java.util.Map;
public class Compensate extends Thread {<!-- -->
private static final String TAG = "Head Compensate";
private point tl;
private point br;
private int width;
private int height;
private Application application;
private Context context;
public Compensate(Map<String, Object> map, Application application, Context context) {<!-- -->
this.br = (Point) map.get("br");
this.tl = (Point) map.get("tl");
this.height = (int) map.get("height");
this.width = (int) map.get("width");
this.application = application;
this.context = context;
}
@Override
public void run() {<!-- -->
super.run();
//upper right
double tlx = tl.x;
double tly = tl.y;
//lower left
double brx = br.x;
double bry = br.y;
//Face abscissa reference position
double f_x = (width - (brx - tlx)) / 2;
//Face vertical coordinate reference position
double f_y = (height - (bry - tly)) / 2;
//Actual face abscissa
double s_f_x = tl.x;
//The actual vertical coordinate of the face
double s_f_y = tl.y;
Log.d(TAG, "run: face abscissa reference position:" + f_x);
Log.d(TAG, "run: actual face abscissa:" + s_f_x);
String msg = "stopMotor";
if (f_x - s_f_x > 100) {<!-- -->
Log.d(TAG, "HeadCompensate: " + "The abscissa reference position is greater than the actual abscissa, and the abscissa position is compensated to the right" + (f_x - s_f_x));
//Control the motor drive to perform rightward compensation operation;
msg = "0100";
}
if (s_f_x - f_x > 100) {<!-- -->
Log.d(TAG, "HeadCompensate: " + "The abscissa reference position is greater than the actual abscissa, and the abscissa position is compensated to the left" + (s_f_x - f_x));
//Control the motor drive to perform left compensation operation;
msg = "0101";
}
Log.d(TAG, "run: face ordinate reference position:" + f_y);
Log.d(TAG, "run: actual face ordinate:" + s_f_y);
if (f_y - s_f_y > 100) {<!-- -->
Log.d(TAG, "HeadCompensate: " + "The abscissa reference position is greater than the actual abscissa, perform downward compensation of the abscissa position" + (f_y - s_f_y));
//Control the motor drive to perform downward compensation operation;
msg = "1000";
}
if (s_f_y - f_y > 100) {<!-- -->
Log.d(TAG, "HeadCompensate: " + "The abscissa reference position is greater than the actual abscissa, perform upward compensation of the abscissa position" + (s_f_y - f_y));
//Control the motor drive to perform upward compensation operation;
msg = "1001";
}
Ch34 ch34 = new Ch34(application, context, msg);
ch34.start();
}
}
CH340 serial communication class
- I checked a lot of information on the Internet here, and it was inconsistent with the jar package I downloaded. It may be that the official has updated the new jar package. Finally, let me complain about the following official documentation. Only the description of each method does not explain the calling process. I finally realized the communication between the host computer and the slave computer by guessing. Whether there are any bugs needs to be tested.
import android.app.Application;
import android.content.Context;
import android.hardware.usb.UsbDevice;
import android.util.Log;
import com.xuye.openeye.APP;
import cn.wch.uartlib.WCHUARTManager;
import cn.wch.uartlib.callback.IDataCallback;
import static android.content.ContentValues.TAG;
public class Ch34 extends Thread{<!-- -->
private final Application application;
private final Context context;
private String msg;
public Ch34(Application application,Context context,String msg){<!-- -->
this.application=application;
this.context=context;
this.msg=msg;
}
private final IDataCallback callback= (i, bytes, i1) -> {<!-- -->
Log.i(TAG, "onData: " + i);
Log.i(TAG, "onData: " + i1);
Log.i(TAG, "onData: " + new String(bytes));
};
private void sendData(){<!-- -->
//A /n newline character must be added to the end of the string, otherwise uno cannot trigger the string reception completion flag.
msg + ="\
";
APP.manager= WCHUARTManager.getInstance();
APP.manager.init(application);
try {<!-- -->
UsbDevice device=APP.manager.enumDevice().get(0);
APP.manager.requestPermission(context,device);
if(!APP.manager.isConnected(device)){<!-- -->
APP.manager.openDevice(device);
}
APP.manager.setSerialParameter(device,0,115200,8,1,0,false);
APP.manager.registerDataCallback(device,callback);
APP.manager.writeData(device,0,msg.getBytes(),msg.length(),1000);
} catch (Exception e) {<!-- -->
e.printStackTrace();
}
}
@Override
public void run(){<!-- -->
super.run();
sendData();
}
}
Global CH340Driver object
import android.app.Application;
import cn.wch.uartlib.WCHUARTManager;
public class APP extends Application {<!-- -->
public static WCHUARTManager manager;
}