diven
3 years ago
7 changed files with 397 additions and 0 deletions
@ -0,0 +1,41 @@ |
|||||
|
package com.visual.open.anpr.core.domain; |
||||
|
|
||||
|
import org.opencv.core.Mat; |
||||
|
|
||||
|
public class BorderMat { |
||||
|
/**图片数据*/ |
||||
|
public Mat mat; |
||||
|
/**图片的缩放比率**/ |
||||
|
public float scale; |
||||
|
/**往上补充的像素宽度**/ |
||||
|
public int top; |
||||
|
/**往下补充的像素宽度**/ |
||||
|
public int bottom; |
||||
|
/**往左补充的像素宽度**/ |
||||
|
public int left; |
||||
|
/**往右补充的像素宽度**/ |
||||
|
public int right; |
||||
|
|
||||
|
public BorderMat(Mat mat, float scale, int top, int bottom, int left, int right) { |
||||
|
this.mat = mat; |
||||
|
this.scale = scale; |
||||
|
this.top = top; |
||||
|
this.bottom = bottom; |
||||
|
this.left = left; |
||||
|
this.right = right; |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* 释放资源 |
||||
|
*/ |
||||
|
public void release(){ |
||||
|
if(this.mat != null){ |
||||
|
try { |
||||
|
this.mat.release(); |
||||
|
this.mat = null; |
||||
|
}catch (Exception e){ |
||||
|
e.printStackTrace(); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,249 @@ |
|||||
|
package com.visual.open.anpr.core.models; |
||||
|
|
||||
|
import ai.onnxruntime.OnnxTensor; |
||||
|
import ai.onnxruntime.OrtSession; |
||||
|
import com.visual.open.anpr.core.base.BaseOnnxInfer; |
||||
|
import com.visual.open.anpr.core.base.PlateDetection; |
||||
|
import com.visual.open.anpr.core.domain.ImageMat; |
||||
|
import com.visual.open.anpr.core.domain.BorderMat; |
||||
|
import com.visual.open.anpr.core.domain.PlateInfo; |
||||
|
import com.visual.open.anpr.core.utils.ReleaseUtil; |
||||
|
import org.opencv.core.*; |
||||
|
import org.opencv.imgproc.Imgproc; |
||||
|
|
||||
|
import java.util.*; |
||||
|
import java.util.stream.Collectors; |
||||
|
|
||||
|
public class TorchPlateDetection extends BaseOnnxInfer implements PlateDetection { |
||||
|
private static int imageWidth = 640; |
||||
|
private static int imageHeight= 640; |
||||
|
private static Scalar border = new Scalar(114, 114, 114); |
||||
|
|
||||
|
public TorchPlateDetection(String modelPath, int threads) { |
||||
|
super(modelPath, threads); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public List<PlateInfo> inference(ImageMat image, float scoreTh, float iouTh, Map<String, Object> params) { |
||||
|
OnnxTensor tensor = null; |
||||
|
OrtSession.Result output = null; |
||||
|
BorderMat makeBorderMat = null; |
||||
|
ImageMat imageMat = image.clone(); |
||||
|
try { |
||||
|
//对图像进行标准宽高的处理
|
||||
|
makeBorderMat = resizeAndMakeBorderMat(imageMat.toCvMat(), imageWidth, imageHeight); |
||||
|
//转换数据为张量
|
||||
|
tensor = ImageMat.fromCVMat(makeBorderMat.mat) |
||||
|
.blobFromImageAndDoReleaseMat(1.0/255, new Scalar(0, 0, 0), true) |
||||
|
.to4dFloatOnnxTensorAndNoReleaseMat(new float[]{1,1,1},true); |
||||
|
//ONNX推理
|
||||
|
output = getSession().run(Collections.singletonMap(getInputName(), tensor)); |
||||
|
float[][][] result = (float[][][]) output.get(0).getValue(); |
||||
|
//候选框的处理
|
||||
|
List<float[]> boxes = filterCandidateBoxes(result[0], scoreTh, iouTh, params); |
||||
|
//根据入模一起对图片的处理参数对box进行还原
|
||||
|
List<float[]> restoreBoxes = restoreBoxes(boxes, makeBorderMat); |
||||
|
//模型后处理，转换为标准的结构化模型
|
||||
|
List<PlateInfo> plateInfos = new ArrayList<>(); |
||||
|
for (float[] item : restoreBoxes){ |
||||
|
//数据模型转换
|
||||
|
PlateInfo plateInfo = PlateInfo.build(item[4], PlateInfo.PlateBox.build( |
||||
|
PlateInfo.Point.build( |
||||
|
clip(item[5], 0, imageMat.getWidth()), |
||||
|
clip(item[6], 0, imageMat.getHeight())), |
||||
|
PlateInfo.Point.build( |
||||
|
clip(item[7], 0, imageMat.getWidth()), |
||||
|
clip(item[8], 0, imageMat.getHeight())), |
||||
|
PlateInfo.Point.build( |
||||
|
clip(item[9], 0, imageMat.getWidth()), |
||||
|
clip(item[10], 0, imageMat.getHeight())), |
||||
|
PlateInfo.Point.build( |
||||
|
clip(item[11], 0, imageMat.getWidth()), |
||||
|
clip(item[12], 0, imageMat.getHeight())) |
||||
|
)); |
||||
|
plateInfos.add(plateInfo); |
||||
|
} |
||||
|
//返回
|
||||
|
return plateInfos; |
||||
|
}catch (Exception e){ |
||||
|
//抛出异常
|
||||
|
throw new RuntimeException(e); |
||||
|
}finally { |
||||
|
//释放资源
|
||||
|
if(null != tensor){ |
||||
|
ReleaseUtil.release(tensor); |
||||
|
} |
||||
|
if(null != output){ |
||||
|
ReleaseUtil.release(output); |
||||
|
} |
||||
|
if(null != makeBorderMat){ |
||||
|
ReleaseUtil.release(makeBorderMat); |
||||
|
} |
||||
|
if(null != imageMat){ |
||||
|
ReleaseUtil.release(imageMat); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* 候选框的处理 |
||||
|
* @param result 预测结果 |
||||
|
* @param scoreTh 候选框的分数阈值 |
||||
|
* @param iouTh 重叠比率 |
||||
|
* @param params 额外的参数 |
||||
|
* @return |
||||
|
*/ |
||||
|
private static List<float[]> filterCandidateBoxes(float[][] result, float scoreTh, float iouTh, Map<String, Object> params){ |
||||
|
//对预测的候选框进行预处理
|
||||
|
List<float[]> boxesForPretreatment = pretreatmentBoxes(result, scoreTh); |
||||
|
//根据iou进行车牌框过滤
|
||||
|
List<float[]> boxesForNms = filterByNmsForIou(boxesForPretreatment, iouTh); |
||||
|
//返回
|
||||
|
return boxesForNms; |
||||
|
} |
||||
|
|
||||
|
|
||||
|
/** |
||||
|
* 对图像进行标准宽高的处理 |
||||
|
* @param image 原始图片 |
||||
|
* @param targetWidth 目标图片的宽度 |
||||
|
* @param targetHeight 目标图片的高度 |
||||
|
* @return |
||||
|
*/ |
||||
|
private static BorderMat resizeAndMakeBorderMat(Mat image, int targetWidth, int targetHeight){ |
||||
|
Mat resizeDst = null; |
||||
|
try { |
||||
|
int imageWidth = image.width(); |
||||
|
int imageHeight = image.height(); |
||||
|
float scaling = Math.min(1.0f * targetHeight / imageHeight, 1.0f * targetWidth / imageWidth); |
||||
|
int newHeight = Double.valueOf(imageHeight * scaling).intValue(); |
||||
|
int newWidth = Double.valueOf(imageWidth * scaling).intValue(); |
||||
|
int topOffset = Double.valueOf((targetHeight - newHeight ) / 2.0).intValue(); |
||||
|
int leftOffset = Double.valueOf((targetWidth-newWidth) / 2.0).intValue(); |
||||
|
int bottomOffset = targetHeight - newHeight -topOffset ; |
||||
|
int rightOffset = targetWidth - newWidth-leftOffset ; |
||||
|
resizeDst = new Mat(); |
||||
|
Imgproc.resize(image, resizeDst, new Size(newWidth,newHeight ), 0, 0, Imgproc.INTER_AREA); |
||||
|
Mat res = new Mat(); |
||||
|
Core.copyMakeBorder(resizeDst, res, topOffset, bottomOffset, leftOffset, rightOffset, Core.BORDER_CONSTANT, border); |
||||
|
return new BorderMat(res, scaling, topOffset, bottomOffset, leftOffset, rightOffset); |
||||
|
}finally { |
||||
|
ReleaseUtil.release(resizeDst); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* 对预测的候选框进行预处理 |
||||
|
* @param result 模型预测的候选框 |
||||
|
* @param scoreThresh 候选框的分数阈值 |
||||
|
* @return 处理后的待选框 |
||||
|
*/ |
||||
|
private static List<float[]> pretreatmentBoxes(float[][] result, float scoreThresh){ |
||||
|
return |
||||
|
Arrays.stream(result) |
||||
|
.filter(item -> item[4] > scoreThresh) |
||||
|
.map(item -> { |
||||
|
float[] temp = new float[14]; |
||||
|
//计算分数
|
||||
|
item[13] = item[13] * item[4]; |
||||
|
item[14] = item[14] * item[4]; |
||||
|
//计算坐标
|
||||
|
temp[0] = item[0] - item[2] / 2; |
||||
|
temp[1] = item[1] - item[3] / 2; |
||||
|
temp[2] = item[0] + item[2] / 2; |
||||
|
temp[3] = item[1] + item[3] / 2; |
||||
|
//计算车牌的预测分数
|
||||
|
temp[4] = Math.max(item[13], item[14]); |
||||
|
//标记点数据
|
||||
|
temp[5] = item[5]; |
||||
|
temp[6] = item[6]; |
||||
|
temp[7] = item[7]; |
||||
|
temp[8] = item[8]; |
||||
|
temp[9] = item[9]; |
||||
|
temp[10] = item[10]; |
||||
|
temp[11] = item[11]; |
||||
|
temp[12] = item[12]; |
||||
|
//计算是双层还是单层车牌
|
||||
|
temp[13] = item[13] >= item[14] ? 0 : 1; |
||||
|
return temp; |
||||
|
}) |
||||
|
.sorted((a, b) -> Float.compare(b[4], a[4])) |
||||
|
.collect(Collectors.toList()); |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* 根据iou进行车牌框过滤 |
||||
|
* @param boxes 待处理的boxes |
||||
|
* @param iouTh 重叠比率 |
||||
|
* @return 过滤后的车牌坐标 |
||||
|
*/ |
||||
|
private static List<float[]> filterByNmsForIou(List<float[]>boxes, float iouTh){ |
||||
|
List<float[]> result = new ArrayList<>(); |
||||
|
while(!boxes.isEmpty()){ |
||||
|
Iterator<float[]> iterator = boxes.iterator(); |
||||
|
//获取第一个元素，并删除元素
|
||||
|
float[] firstFace = iterator.next(); |
||||
|
iterator.remove(); |
||||
|
//对比后面元素与第一个元素之间的iou
|
||||
|
while (iterator.hasNext()) { |
||||
|
float[] nextFace = iterator.next(); |
||||
|
float x1=Math.max(firstFace[0], nextFace[0]); |
||||
|
float y1=Math.max(firstFace[1], nextFace[1]); |
||||
|
float x2=Math.min(firstFace[2], nextFace[2]); |
||||
|
float y2=Math.min(firstFace[3], nextFace[3]); |
||||
|
float w = Math.max(0, x2-x1); |
||||
|
float h = Math.max(0, y2-y1); |
||||
|
float inter_area = w * h; |
||||
|
float union_area = (firstFace[2] - firstFace[0]) * (firstFace[3] - firstFace[1]) + |
||||
|
(nextFace[2] - nextFace[0]) * (nextFace[3] - nextFace[1]); |
||||
|
float iou = inter_area/(union_area-inter_area); |
||||
|
if(iou >= iouTh){ |
||||
|
iterator.remove(); |
||||
|
} |
||||
|
} |
||||
|
result.add(firstFace); |
||||
|
} |
||||
|
return result; |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* 根据入模一起对图片的处理参数对box进行还原 |
||||
|
* @param boxes 候选框 |
||||
|
* @param border 边框及缩放信息 |
||||
|
* @return |
||||
|
*/ |
||||
|
private static List<float[]> restoreBoxes(List<float[]>boxes, BorderMat border){ |
||||
|
return boxes.stream().peek(item -> { |
||||
|
item[0] = (item[0] - border.left) / border.scale; |
||||
|
item[2] = (item[2] - border.left) / border.scale; |
||||
|
item[5] = (item[5] - border.left) / border.scale; |
||||
|
item[7] = (item[7] - border.left) / border.scale; |
||||
|
item[9] = (item[9] - border.left) / border.scale; |
||||
|
item[11] = (item[11] - border.left) / border.scale; |
||||
|
|
||||
|
item[1] = (item[1] - border.top) / border.scale; |
||||
|
item[3] = (item[3] - border.top) / border.scale; |
||||
|
item[6] = (item[6] - border.top) / border.scale; |
||||
|
item[8] = (item[8] - border.top) / border.scale; |
||||
|
item[10] = (item[10] - border.top) / border.scale; |
||||
|
item[12] = (item[12] - border.top) / border.scale; |
||||
|
}).collect(Collectors.toList()); |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* 边框数据清洗 |
||||
|
* @param value |
||||
|
* @param min |
||||
|
* @param max |
||||
|
* @return |
||||
|
*/ |
||||
|
private static int clip(double value, int min, int max){ |
||||
|
if(value > max){ |
||||
|
return max; |
||||
|
} |
||||
|
if(value < min){ |
||||
|
return min; |
||||
|
} |
||||
|
return Double.valueOf(value).intValue(); |
||||
|
} |
||||
|
} |
||||
Binary file not shown.
Binary file not shown.
@ -0,0 +1,50 @@ |
|||||
|
package com.visual.open.anpr.core.models; |
||||
|
|
||||
|
import ai.onnxruntime.OrtEnvironment; |
||||
|
import org.opencv.core.Core; |
||||
|
import org.opencv.core.Mat; |
||||
|
import org.opencv.core.Scalar; |
||||
|
import org.opencv.core.Size; |
||||
|
import org.opencv.imgcodecs.Imgcodecs; |
||||
|
import org.opencv.imgproc.Imgproc; |
||||
|
|
||||
|
public class TestMain01 { |
||||
|
//静态加载动态链接库
|
||||
|
static{ nu.pattern.OpenCV.loadShared(); } |
||||
|
private OrtEnvironment env = OrtEnvironment.getEnvironment(); |
||||
|
|
||||
|
public static void my_letter_box(Mat image, int imageWidth, int imageHeight){ |
||||
|
int w = image.width(); |
||||
|
int h = image.height(); |
||||
|
double r = Math.min(1.0 * imageHeight / h, 1.0 * imageWidth / w); |
||||
|
System.out.println(r); |
||||
|
int new_h = Double.valueOf(h*r).intValue(); |
||||
|
int new_w = Double.valueOf(w*r).intValue(); |
||||
|
int top = Double.valueOf((imageHeight - new_h) / 2.0).intValue(); |
||||
|
int left = Double.valueOf((imageWidth-new_w) / 2.0).intValue(); |
||||
|
System.out.println(top); |
||||
|
System.out.println(left); |
||||
|
|
||||
|
int bottom = imageHeight - new_h-top ; |
||||
|
int right = imageWidth - new_w-left ; |
||||
|
System.out.println(bottom); |
||||
|
System.out.println(right); |
||||
|
|
||||
|
Mat resizeDst = new Mat(); |
||||
|
Imgproc.resize(image, resizeDst, new Size(new_w,new_h), 0, 0, Imgproc.INTER_AREA); |
||||
|
|
||||
|
Mat res = new Mat(); |
||||
|
Core.copyMakeBorder(resizeDst, res, top, bottom, left, right, Core.BORDER_CONSTANT, new Scalar(114,114,114)); |
||||
|
|
||||
|
Imgcodecs.imwrite("res.jpg", res); |
||||
|
} |
||||
|
|
||||
|
public static void main(String[] args) { |
||||
|
String imagePath = "open-anpr-core/src/test/resources/images/imagetmp.jpg"; |
||||
|
|
||||
|
Mat image = Imgcodecs.imread(imagePath); |
||||
|
my_letter_box(image, 640, 640); |
||||
|
|
||||
|
|
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,42 @@ |
|||||
|
package com.visual.open.anpr.core.models; |
||||
|
|
||||
|
import com.visual.open.anpr.core.domain.DrawImage; |
||||
|
import com.visual.open.anpr.core.domain.ImageMat; |
||||
|
import com.visual.open.anpr.core.domain.PlateInfo; |
||||
|
|
||||
|
import java.awt.*; |
||||
|
import java.util.HashMap; |
||||
|
import java.util.List; |
||||
|
|
||||
|
public class TorchPlateDetectionTest { |
||||
|
public static void main(String[] args) { |
||||
|
TorchPlateDetection torchPlateDetection = new TorchPlateDetection("open-anpr-core/src/main/resources/models/plate_detect.onnx", 1); |
||||
|
String imagePath = "/Users/diven/workspace/idea/gitee/open-anpr/open-anpr-core/src/test/resources/images/image003.jpg"; |
||||
|
// String imagePath = "/Users/diven/workspace/pycharm/github/Chinese_license_plate_detection_recognition/imgs3/double_yellow.jpg";
|
||||
|
ImageMat imageMat = ImageMat.fromImage(imagePath); |
||||
|
List<PlateInfo> plateInfos = torchPlateDetection.inference(imageMat, 0.3f,0.5f, new HashMap<>()); |
||||
|
System.out.println(plateInfos); |
||||
|
|
||||
|
DrawImage drawImage = DrawImage.build(imagePath); |
||||
|
for(PlateInfo plateInfo : plateInfos){ |
||||
|
PlateInfo.Point [] points = plateInfo.box.toArray(); |
||||
|
for(int i =0; i< points.length; i++){ |
||||
|
if(i+1 == points.length){ |
||||
|
drawImage.drawLine( |
||||
|
new DrawImage.Point((int)points[i].x, (int)points[i].y), |
||||
|
new DrawImage.Point((int)points[0].x, (int)points[0].y), |
||||
|
2, Color.RED |
||||
|
); |
||||
|
}else{ |
||||
|
drawImage.drawLine( |
||||
|
new DrawImage.Point((int)points[i].x, (int)points[i].y), |
||||
|
new DrawImage.Point((int)points[i+1].x, (int)points[i+1].y), |
||||
|
2, Color.RED |
||||
|
); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
ImageMat.fromCVMat(drawImage.toMat()).imShow(); |
||||
|
} |
||||
|
|
||||
|
} |
||||
Loading…
Reference in new issue