diff --git a/open-anpr-core/src/main/java/com/visual/open/anpr/core/domain/BorderMat.java b/open-anpr-core/src/main/java/com/visual/open/anpr/core/domain/BorderMat.java new file mode 100644 index 0000000..df845be --- /dev/null +++ b/open-anpr-core/src/main/java/com/visual/open/anpr/core/domain/BorderMat.java @@ -0,0 +1,41 @@ +package com.visual.open.anpr.core.domain; + +import org.opencv.core.Mat; + +public class BorderMat { + /**图片数据*/ + public Mat mat; + /**图片的缩放比率**/ + public float scale; + /**往上补充的像素宽度**/ + public int top; + /**往下补充的像素宽度**/ + public int bottom; + /**往左补充的像素宽度**/ + public int left; + /**往右补充的像素宽度**/ + public int right; + + public BorderMat(Mat mat, float scale, int top, int bottom, int left, int right) { + this.mat = mat; + this.scale = scale; + this.top = top; + this.bottom = bottom; + this.left = left; + this.right = right; + } + + /** + * 释放资源 + */ + public void release(){ + if(this.mat != null){ + try { + this.mat.release(); + this.mat = null; + }catch (Exception e){ + e.printStackTrace(); + } + } + } +} diff --git a/open-anpr-core/src/main/java/com/visual/open/anpr/core/models/TorchPlateDetection.java b/open-anpr-core/src/main/java/com/visual/open/anpr/core/models/TorchPlateDetection.java new file mode 100644 index 0000000..ebf31ca --- /dev/null +++ b/open-anpr-core/src/main/java/com/visual/open/anpr/core/models/TorchPlateDetection.java @@ -0,0 +1,249 @@ +package com.visual.open.anpr.core.models; + +import ai.onnxruntime.OnnxTensor; +import ai.onnxruntime.OrtSession; +import com.visual.open.anpr.core.base.BaseOnnxInfer; +import com.visual.open.anpr.core.base.PlateDetection; +import com.visual.open.anpr.core.domain.ImageMat; +import com.visual.open.anpr.core.domain.BorderMat; +import com.visual.open.anpr.core.domain.PlateInfo; +import com.visual.open.anpr.core.utils.ReleaseUtil; +import org.opencv.core.*; +import org.opencv.imgproc.Imgproc; + +import java.util.*; +import java.util.stream.Collectors; + +public class TorchPlateDetection extends BaseOnnxInfer implements PlateDetection { + private static int imageWidth = 640; + private static int imageHeight= 640; + private static Scalar border = new Scalar(114, 114, 114); + + public TorchPlateDetection(String modelPath, int threads) { + super(modelPath, threads); + } + + @Override + public List inference(ImageMat image, float scoreTh, float iouTh, Map params) { + OnnxTensor tensor = null; + OrtSession.Result output = null; + BorderMat makeBorderMat = null; + ImageMat imageMat = image.clone(); + try { + //对图像进行标准宽高的处理 + makeBorderMat = resizeAndMakeBorderMat(imageMat.toCvMat(), imageWidth, imageHeight); + //转换数据为张量 + tensor = ImageMat.fromCVMat(makeBorderMat.mat) + .blobFromImageAndDoReleaseMat(1.0/255, new Scalar(0, 0, 0), true) + .to4dFloatOnnxTensorAndNoReleaseMat(new float[]{1,1,1},true); + //ONNX推理 + output = getSession().run(Collections.singletonMap(getInputName(), tensor)); + float[][][] result = (float[][][]) output.get(0).getValue(); + //候选框的处理 + List boxes = filterCandidateBoxes(result[0], scoreTh, iouTh, params); + //根据入模一起对图片的处理参数对box进行还原 + List restoreBoxes = restoreBoxes(boxes, makeBorderMat); + //模型后处理，转换为标准的结构化模型 + List plateInfos = new ArrayList<>(); + for (float[] item : restoreBoxes){ + //数据模型转换 + PlateInfo plateInfo = PlateInfo.build(item[4], PlateInfo.PlateBox.build( + PlateInfo.Point.build( + clip(item[5], 0, imageMat.getWidth()), + clip(item[6], 0, imageMat.getHeight())), + PlateInfo.Point.build( + clip(item[7], 0, imageMat.getWidth()), + clip(item[8], 0, imageMat.getHeight())), + PlateInfo.Point.build( + clip(item[9], 0, imageMat.getWidth()), + clip(item[10], 0, imageMat.getHeight())), + PlateInfo.Point.build( + clip(item[11], 0, imageMat.getWidth()), + clip(item[12], 0, imageMat.getHeight())) + )); + plateInfos.add(plateInfo); + } + //返回 + return plateInfos; + }catch (Exception e){ + //抛出异常 + throw new RuntimeException(e); + }finally { + //释放资源 + if(null != tensor){ + ReleaseUtil.release(tensor); + } + if(null != output){ + ReleaseUtil.release(output); + } + if(null != makeBorderMat){ + ReleaseUtil.release(makeBorderMat); + } + if(null != imageMat){ + ReleaseUtil.release(imageMat); + } + } + } + + /** + * 候选框的处理 + * @param result 预测结果 + * @param scoreTh 候选框的分数阈值 + * @param iouTh 重叠比率 + * @param params 额外的参数 + * @return + */ + private static List filterCandidateBoxes(float[][] result, float scoreTh, float iouTh, Map params){ + //对预测的候选框进行预处理 + List boxesForPretreatment = pretreatmentBoxes(result, scoreTh); + //根据iou进行车牌框过滤 + List boxesForNms = filterByNmsForIou(boxesForPretreatment, iouTh); + //返回 + return boxesForNms; + } + + + /** + * 对图像进行标准宽高的处理 + * @param image 原始图片 + * @param targetWidth 目标图片的宽度 + * @param targetHeight 目标图片的高度 + * @return + */ + private static BorderMat resizeAndMakeBorderMat(Mat image, int targetWidth, int targetHeight){ + Mat resizeDst = null; + try { + int imageWidth = image.width(); + int imageHeight = image.height(); + float scaling = Math.min(1.0f * targetHeight / imageHeight, 1.0f * targetWidth / imageWidth); + int newHeight = Double.valueOf(imageHeight * scaling).intValue(); + int newWidth = Double.valueOf(imageWidth * scaling).intValue(); + int topOffset = Double.valueOf((targetHeight - newHeight ) / 2.0).intValue(); + int leftOffset = Double.valueOf((targetWidth-newWidth) / 2.0).intValue(); + int bottomOffset = targetHeight - newHeight -topOffset ; + int rightOffset = targetWidth - newWidth-leftOffset ; + resizeDst = new Mat(); + Imgproc.resize(image, resizeDst, new Size(newWidth,newHeight ), 0, 0, Imgproc.INTER_AREA); + Mat res = new Mat(); + Core.copyMakeBorder(resizeDst, res, topOffset, bottomOffset, leftOffset, rightOffset, Core.BORDER_CONSTANT, border); + return new BorderMat(res, scaling, topOffset, bottomOffset, leftOffset, rightOffset); + }finally { + ReleaseUtil.release(resizeDst); + } + } + + /** + * 对预测的候选框进行预处理 + * @param result 模型预测的候选框 + * @param scoreThresh 候选框的分数阈值 + * @return 处理后的待选框 + */ + private static List pretreatmentBoxes(float[][] result, float scoreThresh){ + return + Arrays.stream(result) + .filter(item -> item[4] > scoreThresh) + .map(item -> { + float[] temp = new float[14]; + //计算分数 + item[13] = item[13] * item[4]; + item[14] = item[14] * item[4]; + //计算坐标 + temp[0] = item[0] - item[2] / 2; + temp[1] = item[1] - item[3] / 2; + temp[2] = item[0] + item[2] / 2; + temp[3] = item[1] + item[3] / 2; + //计算车牌的预测分数 + temp[4] = Math.max(item[13], item[14]); + //标记点数据 + temp[5] = item[5]; + temp[6] = item[6]; + temp[7] = item[7]; + temp[8] = item[8]; + temp[9] = item[9]; + temp[10] = item[10]; + temp[11] = item[11]; + temp[12] = item[12]; + //计算是双层还是单层车牌 + temp[13] = item[13] >= item[14] ? 0 : 1; + return temp; + }) + .sorted((a, b) -> Float.compare(b[4], a[4])) + .collect(Collectors.toList()); + } + + /** + * 根据iou进行车牌框过滤 + * @param boxes 待处理的boxes + * @param iouTh 重叠比率 + * @return 过滤后的车牌坐标 + */ + private static List filterByNmsForIou(Listboxes, float iouTh){ + List result = new ArrayList<>(); + while(!boxes.isEmpty()){ + Iterator iterator = boxes.iterator(); + //获取第一个元素，并删除元素 + float[] firstFace = iterator.next(); + iterator.remove(); + //对比后面元素与第一个元素之间的iou + while (iterator.hasNext()) { + float[] nextFace = iterator.next(); + float x1=Math.max(firstFace[0], nextFace[0]); + float y1=Math.max(firstFace[1], nextFace[1]); + float x2=Math.min(firstFace[2], nextFace[2]); + float y2=Math.min(firstFace[3], nextFace[3]); + float w = Math.max(0, x2-x1); + float h = Math.max(0, y2-y1); + float inter_area = w * h; + float union_area = (firstFace[2] - firstFace[0]) * (firstFace[3] - firstFace[1]) + + (nextFace[2] - nextFace[0]) * (nextFace[3] - nextFace[1]); + float iou = inter_area/(union_area-inter_area); + if(iou >= iouTh){ + iterator.remove(); + } + } + result.add(firstFace); + } + return result; + } + + /** + * 根据入模一起对图片的处理参数对box进行还原 + * @param boxes 候选框 + * @param border 边框及缩放信息 + * @return + */ + private static List restoreBoxes(Listboxes, BorderMat border){ + return boxes.stream().peek(item -> { + item[0] = (item[0] - border.left) / border.scale; + item[2] = (item[2] - border.left) / border.scale; + item[5] = (item[5] - border.left) / border.scale; + item[7] = (item[7] - border.left) / border.scale; + item[9] = (item[9] - border.left) / border.scale; + item[11] = (item[11] - border.left) / border.scale; + + item[1] = (item[1] - border.top) / border.scale; + item[3] = (item[3] - border.top) / border.scale; + item[6] = (item[6] - border.top) / border.scale; + item[8] = (item[8] - border.top) / border.scale; + item[10] = (item[10] - border.top) / border.scale; + item[12] = (item[12] - border.top) / border.scale; + }).collect(Collectors.toList()); + } + + /** + * 边框数据清洗 + * @param value + * @param min + * @param max + * @return + */ + private static int clip(double value, int min, int max){ + if(value > max){ + return max; + } + if(value < min){ + return min; + } + return Double.valueOf(value).intValue(); + } +} diff --git a/open-anpr-core/src/main/java/com/visual/open/anpr/core/utils/ReleaseUtil.java b/open-anpr-core/src/main/java/com/visual/open/anpr/core/utils/ReleaseUtil.java index 5fdba47..01d0fe4 100644 --- a/open-anpr-core/src/main/java/com/visual/open/anpr/core/utils/ReleaseUtil.java +++ b/open-anpr-core/src/main/java/com/visual/open/anpr/core/utils/ReleaseUtil.java @@ -2,6 +2,7 @@ package com.visual.open.anpr.core.utils; import ai.onnxruntime.OnnxTensor; import ai.onnxruntime.OrtSession; +import com.visual.open.anpr.core.domain.BorderMat; import com.visual.open.anpr.core.domain.ImageMat; import org.opencv.core.Mat; @@ -36,6 +37,20 @@ public class ReleaseUtil { } } + public static void release(BorderMat ...borderMats){ + for(BorderMat borderMat : borderMats){ + if(null != borderMat){ + try { + borderMat.release(); + }catch (Exception e){ + e.printStackTrace(); + }finally { + borderMat = null; + } + } + } + } + public static void release(OnnxTensor ...tensors){ if(null == tensors || tensors.length == 0){ return; diff --git a/open-anpr-core/src/main/resources/models/plate_detect.onnx b/open-anpr-core/src/main/resources/models/plate_detect.onnx new file mode 100644 index 0000000..0513eff Binary files /dev/null and b/open-anpr-core/src/main/resources/models/plate_detect.onnx differ diff --git a/open-anpr-core/src/main/resources/models/plate_rec_color.onnx b/open-anpr-core/src/main/resources/models/plate_rec_color.onnx new file mode 100644 index 0000000..ebd7e3e Binary files /dev/null and b/open-anpr-core/src/main/resources/models/plate_rec_color.onnx differ diff --git a/open-anpr-core/src/test/java/com/visual/open/anpr/core/models/TestMain01.java b/open-anpr-core/src/test/java/com/visual/open/anpr/core/models/TestMain01.java new file mode 100644 index 0000000..e15ecd5 --- /dev/null +++ b/open-anpr-core/src/test/java/com/visual/open/anpr/core/models/TestMain01.java @@ -0,0 +1,50 @@ +package com.visual.open.anpr.core.models; + +import ai.onnxruntime.OrtEnvironment; +import org.opencv.core.Core; +import org.opencv.core.Mat; +import org.opencv.core.Scalar; +import org.opencv.core.Size; +import org.opencv.imgcodecs.Imgcodecs; +import org.opencv.imgproc.Imgproc; + +public class TestMain01 { + //静态加载动态链接库 + static{ nu.pattern.OpenCV.loadShared(); } + private OrtEnvironment env = OrtEnvironment.getEnvironment(); + + public static void my_letter_box(Mat image, int imageWidth, int imageHeight){ + int w = image.width(); + int h = image.height(); + double r = Math.min(1.0 * imageHeight / h, 1.0 * imageWidth / w); + System.out.println(r); + int new_h = Double.valueOf(h*r).intValue(); + int new_w = Double.valueOf(w*r).intValue(); + int top = Double.valueOf((imageHeight - new_h) / 2.0).intValue(); + int left = Double.valueOf((imageWidth-new_w) / 2.0).intValue(); + System.out.println(top); + System.out.println(left); + + int bottom = imageHeight - new_h-top ; + int right = imageWidth - new_w-left ; + System.out.println(bottom); + System.out.println(right); + + Mat resizeDst = new Mat(); + Imgproc.resize(image, resizeDst, new Size(new_w,new_h), 0, 0, Imgproc.INTER_AREA); + + Mat res = new Mat(); + Core.copyMakeBorder(resizeDst, res, top, bottom, left, right, Core.BORDER_CONSTANT, new Scalar(114,114,114)); + + Imgcodecs.imwrite("res.jpg", res); + } + + public static void main(String[] args) { + String imagePath = "open-anpr-core/src/test/resources/images/imagetmp.jpg"; + + Mat image = Imgcodecs.imread(imagePath); + my_letter_box(image, 640, 640); + + + } +} diff --git a/open-anpr-core/src/test/java/com/visual/open/anpr/core/models/TorchPlateDetectionTest.java b/open-anpr-core/src/test/java/com/visual/open/anpr/core/models/TorchPlateDetectionTest.java new file mode 100644 index 0000000..b49ac1b --- /dev/null +++ b/open-anpr-core/src/test/java/com/visual/open/anpr/core/models/TorchPlateDetectionTest.java @@ -0,0 +1,42 @@ +package com.visual.open.anpr.core.models; + +import com.visual.open.anpr.core.domain.DrawImage; +import com.visual.open.anpr.core.domain.ImageMat; +import com.visual.open.anpr.core.domain.PlateInfo; + +import java.awt.*; +import java.util.HashMap; +import java.util.List; + +public class TorchPlateDetectionTest { + public static void main(String[] args) { + TorchPlateDetection torchPlateDetection = new TorchPlateDetection("open-anpr-core/src/main/resources/models/plate_detect.onnx", 1); + String imagePath = "/Users/diven/workspace/idea/gitee/open-anpr/open-anpr-core/src/test/resources/images/image003.jpg"; +// String imagePath = "/Users/diven/workspace/pycharm/github/Chinese_license_plate_detection_recognition/imgs3/double_yellow.jpg"; + ImageMat imageMat = ImageMat.fromImage(imagePath); + List plateInfos = torchPlateDetection.inference(imageMat, 0.3f,0.5f, new HashMap<>()); + System.out.println(plateInfos); + + DrawImage drawImage = DrawImage.build(imagePath); + for(PlateInfo plateInfo : plateInfos){ + PlateInfo.Point [] points = plateInfo.box.toArray(); + for(int i =0; i< points.length; i++){ + if(i+1 == points.length){ + drawImage.drawLine( + new DrawImage.Point((int)points[i].x, (int)points[i].y), + new DrawImage.Point((int)points[0].x, (int)points[0].y), + 2, Color.RED + ); + }else{ + drawImage.drawLine( + new DrawImage.Point((int)points[i].x, (int)points[i].y), + new DrawImage.Point((int)points[i+1].x, (int)points[i+1].y), + 2, Color.RED + ); + } + } + } + ImageMat.fromCVMat(drawImage.toMat()).imShow(); + } + +}