Android OCR Recognition for Passport MRZ

Previously, I wrote a post about how to recognize passport MRZ from an image file using C++. In this post, I will create a more productive Android app, which utilizes camera to recognize MRZ.

Requirements

Getting Started with Android Camera View

Android camera2 API is a replacement for the older Camera API. Since it is more powerful and flexible, Android officially recommends it. However, it is still a bit complicated to use. To simplify the programming progress, I will use Dynamsoft Camera Enhancer which not only wraps the camera2 API, but also provides image processing functions to optimize camera frames.

# project/build.gradle
allprojects {
repositories {
maven {
url "https://download.dynamsoft.com/maven/dce/aar"
}
}
}

# app/build.gradle
dependencies {
implementation 'com.dynamsoft:dynamsoftcameraenhancer:1.0.3@aar'
}
<com.dynamsoft.dce.CameraView
android:id="@+id/cameraView"
android:layout_width="match_parent"
android:layout_height="match_parent" >
</com.dynamsoft.dce.CameraView>
@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_main);
initDCE();
}

private void initDCE() {
cameraView = findViewById(R.id.cameraView);
mCameraEnhancer = new CameraEnhancer(MainActivity.this);
mCameraEnhancer.addCameraView(cameraView);
com.dynamsoft.dce.DMDLSConnectionParameters info = new com.dynamsoft.dce.DMDLSConnectionParameters();
info.organizationID = "200001"; // Get the organization ID from https://www.dynamsoft.com/customer/license/trialLicense?product=dce
mCameraEnhancer.initLicenseFromDLS(info,new CameraDLSLicenseVerificationListener() {
@Override
public void DLSLicenseVerificationCallback(boolean isSuccess, Exception error) {
if(!isSuccess){
error.printStackTrace();
}
}
});
//Turn on the camera
mCameraEnhancer.setCameraDesiredState(CameraState.CAMERA_STATE_ON);
}
@Override
protected void onPause() {
super.onPause();
mCameraEnhancer.pauseCamera();
}

@Override
protected void onResume() {
super.onResume();
mCameraEnhancer.resumeCamera();
mProgressBar.setVisibility(View.GONE);
}

@Override
protected void onStart() {
super.onStart();
mCameraEnhancer.startScanning();
}

@Override
protected void onStop() {
super.onStop();
mCameraEnhancer.stopScanning();
}

Recognizing Passport MRZ by Taking a Photo

Once the camera view is ready, we can start integrating OCR SDK into the app. Similar to the way of adding the dependent library of camera SDK, we add the dependent library of OCR SDK to the project:

# project/build.gradle
allprojects {
repositories {
maven {
url "http://download2.dynamsoft.com/maven/dlr/aar"
}
}
}

# app/build.gradle
dependencies {
implementation "com.dynamsoft:dynamsoftlabelrecognition:1.2.1@aar"
}
private void initDLR() {
try {
mRecognition = new LabelRecognition();
DMLTSConnectionParameters parameters = new DMLTSConnectionParameters();
// The organization id 200001 here will grant you a public trial license good for 7 days.
// After that, please visit: https://www.dynamsoft.com/customer/license/trialLicense?product=dlr
// to request for 30 days extension.
parameters.organizationID = "200001";
mRecognition.initLicenseFromLTS(parameters, new DLRLTSLicenseVerificationListener() {
@Override
public void LTSLicenseVerificationCallback(boolean b, final Exception e) {
if (!b) {
e.printStackTrace();
}
}
});
loadModel();
} catch (Exception e) {
Toast.makeText(this, e.getMessage(), Toast.LENGTH_SHORT).show();
e.printStackTrace();
}
}

private void loadModel() {
try {
String[] fileNames = {"NumberUppercase","NumberUppercase_Assist_1lIJ","NumberUppercase_Assist_8B","NumberUppercase_Assist_8BHR","NumberUppercase_Assist_number","NumberUppercase_Assist_O0DQ","NumberUppercase_Assist_upcase"};
for(int i = 0;i<fileNames.length;i++) {
AssetManager manager = getAssets();
InputStream isPrototxt = manager.open("CharacterModel/"+fileNames[i]+".prototxt");
byte[] prototxt = new byte[isPrototxt.available()];
isPrototxt.read(prototxt);
isPrototxt.close();
InputStream isCharacterModel = manager.open("CharacterModel/"+fileNames[i]+".caffemodel");
byte[] characterModel = new byte[isCharacterModel.available()];
isCharacterModel.read(characterModel);
isCharacterModel.close();
InputStream isTxt = manager.open("CharacterModel/"+fileNames[i]+".txt");
byte[] txt = new byte[isTxt.available()];
isTxt.read(txt);
isTxt.close();
mRecognition.appendCharacterModelBuffer(fileNames[i], prototxt, txt, characterModel);
}

StringBuilder stringBuilder = new StringBuilder();
try {
AssetManager manager = getAssets();
BufferedReader bf = new BufferedReader(new InputStreamReader(
manager.open("wholeImgMRZTemplate.json")));
String line;
while ((line = bf.readLine()) != null) {
stringBuilder.append(line);
}
} catch (IOException e) {
e.printStackTrace();
}

mRecognition.appendSettingsFromString(stringBuilder.toString());

} catch (Exception e) {
e.printStackTrace();
}
}
public static byte[] rotateGrayscale90(byte[] data, int width, int height)
{
byte [] grayscale = new byte[width * height];
int index = 0;
for(int i = 0; i < width; i++)
{
for(int j = height - 1; j >= 0; j--)
{
grayscale[index] = data[j * width + i];
index += 1;
}
}

return grayscale;
}
<com.google.android.material.floatingactionbutton.FloatingActionButton
android:id="@+id/fab"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_gravity="end|bottom"
android:layout_margin="16dp"
android:contentDescription="recognize"
android:src="@android:drawable/ic_menu_camera"
tools:ignore="MissingConstraints" />
static String parse(String line1, String line2) {
// https://en.wikipedia.org/wiki/Machine-readable_passport
String result = "";
// Type
String tmp = "Type: ";
tmp += line1.charAt(0);
result += tmp + "\n\n";

// Issuing country
tmp = "Issuing country: ";
tmp += line1.substring(2, 5);
result += tmp + "\n\n";

// Surname
int index = 5;
tmp = "Surname: ";
for (; index < 44; index++) {
if (line1.charAt(index) != '<') {
tmp += line1.charAt(index);
} else {
break;
}
}
result += tmp + "\n\n";

// Given names
tmp = "Given Names: ";
index += 2;
for (; index < 44; index++) {
if (line1.charAt(index) != '<') {
tmp += line1.charAt(index);
} else {
tmp += ' ';
}
}
result += tmp + "\n\n";

// Passport number
tmp = "Passport number: ";
index = 0;
for (; index < 9; index++) {
if (line2.charAt(index) != '<') {
tmp += line2.charAt(index);
} else {
break;
}
}
result += tmp + "\n\n";

// Nationality
tmp = "Nationality: ";
tmp += line2.substring(10, 13);
result += tmp + "\n\n";

// Date of birth
tmp = line2.substring(13, 19);
tmp = tmp.substring(0, 2) +
'/' +
tmp.substring(2, 4) +
'/' +
tmp.substring(4, 6);
tmp = "Date of birth (YYMMDD): " + tmp;
result += tmp + "\n\n";

// Sex
tmp = "Sex: ";
tmp += line2.charAt(20);
result += tmp + "\n\n";

// Expiration date of passport
tmp = line2.substring(21, 27);
tmp = tmp.substring(0, 2) +
'/' +
tmp.substring(2, 4) +
'/' +
tmp.substring(4, 6);
tmp = "Expiration date of passport (YYMMDD): " + tmp;
result += tmp + "\n\n";

// Personal number
if (line2.charAt(28) != '<') {
tmp = "Personal number: ";
for (index = 28; index < 42; index++) {
if (line2.charAt(index) != '<') {
tmp += line2.charAt(index);
} else {
break;
}
}
result += tmp + "\n\n";
}

return result;
}
public static void saveFrame(byte[] data, int width, int height, String path) {
Bitmap bitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ALPHA_8);
bitmap.copyPixelsFromBuffer(ByteBuffer.wrap(data));
try {
FileOutputStream out = new FileOutputStream(path);
bitmap.compress(Bitmap.CompressFormat.PNG, 100, out);
out.close();
} catch (IOException e) {
e.printStackTrace();
}
}
public class ResultActivity extends AppCompatActivity {

@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);

Intent intent = getIntent();
String message = intent.getStringExtra("path");
String result = intent.getStringExtra("result");

Bitmap bitmap = BitmapFactory.decodeFile(message);
ImageView iv = new ImageView(this);
iv.setImageBitmap(bitmap);
setContentView(iv);

Toast.makeText(this, result, Toast.LENGTH_LONG)
.show();
}
}

Source Code

https://github.com/yushulx/android-passport-mrz

Manager of Dynamsoft Open Source Projects | Tech Lover