diff --git a/open-anpr-core/src/main/java/com/visual/open/anpr/core/domain/DrawImage.java b/open-anpr-core/src/main/java/com/visual/open/anpr/core/domain/DrawImage.java new file mode 100644 index 0000000..6d0da7b --- /dev/null +++ b/open-anpr-core/src/main/java/com/visual/open/anpr/core/domain/DrawImage.java @@ -0,0 +1,131 @@ +package com.visual.open.anpr.core.domain; + +import org.opencv.core.CvType; +import org.opencv.core.Mat; + +import javax.imageio.ImageIO; +import java.awt.*; +import java.awt.image.BufferedImage; +import java.awt.image.DataBufferByte; +import java.io.File; +import java.io.IOException; + + +public class DrawImage { + + private BufferedImage image; + + private DrawImage(BufferedImage image){ + this.image = image; + } + + public static DrawImage build(String image){ + try { + return build(ImageIO.read(new File(image))); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public static DrawImage build(BufferedImage image){ + return new DrawImage(image); + } + + private static int getLength(String text) { + int textLength = text.length(); + int length = textLength; + for (int i = 0; i < textLength; i++) { + if (String.valueOf(text.charAt(i)).getBytes().length > 1) { + length++; + } + } + return (length % 2 == 0) ? length / 2 : length / 2 + 1; + } + + public DrawImage drawRect(Rect rect, int lineWidth, Color color){ + Graphics2D g = (Graphics2D)image.getGraphics(); + g.setColor(color); + g.setStroke(new BasicStroke(lineWidth)); + g.drawRect(rect.x, rect.y, rect.width, rect.height); + return this; + } + + public DrawImage drawLine(Point point1, Point point2, int lineWidth, Color color){ + Graphics2D g = (Graphics2D)image.getGraphics(); + g.setColor(color); + g.setStroke(new BasicStroke(lineWidth)); + g.drawLine(point1.x, point1.y, point2.x, point2.y); + return this; + } + + public DrawImage drawText(String text, Point point, int fontSize, Color color){ + Graphics2D g = image.createGraphics(); + g.setColor(color); + g.setFont(new Font("微软雅黑", Font.PLAIN, fontSize)); + g.setComposite(AlphaComposite.getInstance(AlphaComposite.SRC_ATOP, 1)); + int width_1 = fontSize * getLength(text); + int height_1 = fontSize; + int widthDiff = image.getWidth() - width_1; + int heightDiff = image.getHeight() - height_1; + if(point.x < 0){ + point.x = widthDiff / 2; + }else if(point.x > widthDiff){ + point.x = widthDiff; + } + if(point.y < 0){ + point.y = heightDiff / 2; + }else if(point.y > heightDiff){ + point.y = heightDiff; + } + g.drawString(text, point.x, point.y + height_1); + return this; + } + + public Mat toMat(){ + try { + if(image.getType() != BufferedImage.TYPE_3BYTE_BGR){ + BufferedImage temp = new BufferedImage(image.getWidth(), image.getHeight(), BufferedImage.TYPE_3BYTE_BGR); + Graphics2D g = temp.createGraphics(); + try { + g.setComposite(AlphaComposite.Src); + g.drawImage(image, 0, 0, null); + } finally { + g.dispose(); + } + image = temp; + } + byte[] pixels = ((DataBufferByte) image.getRaster().getDataBuffer()).getData(); + Mat mat = Mat.eye(image.getHeight(), image.getWidth(), CvType.CV_8UC3); + mat.put(0, 0, pixels); + return mat; + }catch (Exception e){ + throw new RuntimeException(e); + } + } + + public static class Rect{ + public int x; + public int y; + public int width; + public int height; + + public Rect(int x, int y, int width, int height) { + this.x = x; + this.y = y; + this.width = width; + this.height = height; + } + + } + + public static class Point{ + public int x; + public int y; + + public Point(int x, int y) { + this.x = x; + this.y = y; + } + } + +} diff --git a/open-anpr-core/src/main/java/com/visual/open/anpr/core/domain/PlateInfo.java b/open-anpr-core/src/main/java/com/visual/open/anpr/core/domain/PlateInfo.java index 19a5473..e5edca2 100755 --- a/open-anpr-core/src/main/java/com/visual/open/anpr/core/domain/PlateInfo.java +++ b/open-anpr-core/src/main/java/com/visual/open/anpr/core/domain/PlateInfo.java @@ -158,6 +158,17 @@ public class PlateInfo implements Comparable, Serializable { return new PlateBox((int)x1,(int)y1,(int)x2,(int)y2); } + /** + * 构造一个车牌框 + * @param leftTop 左上角坐标值 + * @param rightTop 右上角坐标 + * @param rightBottom 右下角坐标 + * @param leftBottom 左下角坐标 + */ + public static PlateBox build(Point leftTop, Point rightTop, Point rightBottom, Point leftBottom){ + return new PlateBox(leftTop,rightTop,rightBottom,leftBottom); + } + /** * x的最小坐标 * @return @@ -277,6 +288,15 @@ public class PlateInfo implements Comparable, Serializable { new Point(leftBottom.x + change_x_p2_p4, leftBottom.y + change_y_p2_p4) ); } + + /** + * 转换为数组 + * @return + */ + public Point[] toArray(){ + return new Point[]{leftTop, rightTop, rightBottom, leftBottom}; + } + } } diff --git a/open-anpr-core/src/main/java/com/visual/open/anpr/core/domain/Polygon.java b/open-anpr-core/src/main/java/com/visual/open/anpr/core/domain/Polygon.java new file mode 100644 index 0000000..cbc2ef6 --- /dev/null +++ b/open-anpr-core/src/main/java/com/visual/open/anpr/core/domain/Polygon.java @@ -0,0 +1,39 @@ +package com.visual.open.anpr.core.domain; + +import org.opencv.core.Point; + +import java.util.List; + +public class Polygon { + private List points; + + public Polygon(List points){ + this.points = points; + } + + public double getArea(){ + int i, j; + double area = 0; + for (i = 0; i < this.points.size(); i++) + { + j = (i + 1) % this.points.size(); + area += this.points.get(i).x * this.points.get(j).y; + area -= this.points.get(i).y * this.points.get(j).x; + } + area /= 2; + return Math.abs(area); + } + + public double getLength(){ + double result = 0; + for (int i = 0; i < this.points.size(); i++){ + Point u = points.get(i); + Point v = (i+1==this.points.size()) ? points.get(0): points.get(i+1); + double m = u.x - v.x; + double n = u.y - v.y; + result += Math.sqrt(m * m + n * n); + } + return result; + } + +} diff --git a/open-anpr-core/src/main/java/com/visual/open/anpr/core/models/PaddlePlateDetection.java b/open-anpr-core/src/main/java/com/visual/open/anpr/core/models/PaddlePlateDetection.java index f12aba7..6635818 100644 --- a/open-anpr-core/src/main/java/com/visual/open/anpr/core/models/PaddlePlateDetection.java +++ b/open-anpr-core/src/main/java/com/visual/open/anpr/core/models/PaddlePlateDetection.java @@ -1,19 +1,32 @@ package com.visual.open.anpr.core.models; import ai.onnxruntime.OnnxTensor; +import ai.onnxruntime.OrtSession; import com.visual.open.anpr.core.base.BaseOnnxInfer; import com.visual.open.anpr.core.base.PlateDetection; +import com.visual.open.anpr.core.clipper.Clipper; +import com.visual.open.anpr.core.clipper.ClipperOffset; +import com.visual.open.anpr.core.clipper.Path; +import com.visual.open.anpr.core.clipper.Paths; import com.visual.open.anpr.core.domain.ImageMat; import com.visual.open.anpr.core.domain.PlateInfo; +import com.visual.open.anpr.core.domain.Polygon; import com.visual.open.anpr.core.utils.ReleaseUtil; -import org.opencv.core.Scalar; +import org.opencv.core.*; +import org.opencv.imgproc.Imgproc; -import java.util.List; -import java.util.Map; +import java.util.*; +import java.util.stream.Collectors; public class PaddlePlateDetection extends BaseOnnxInfer implements PlateDetection { - private float std[] = {0.229f, 0.224f, 0.225f}; + private float thresh=0.3f; + private float box_thresh=0.6f; + private int max_candidates=1000; + private float unclip_ratio=1.5f; + private float min_size = 3f; + + private float[] std = {0.229f, 0.224f, 0.225f}; private Scalar mean = new Scalar(0.485, 0.456, 0.406); public PaddlePlateDetection(String modelPath, int threads) { @@ -22,23 +35,196 @@ public class PaddlePlateDetection extends BaseOnnxInfer implements PlateDetectio @Override public List inference(ImageMat image, float scoreTh, float iouTh, Map params) { - ImageMat imageMat = image.clone(); OnnxTensor tensor = null; + OrtSession.Result output = null; + ImageMat imageMat = image.clone(); try { + //前置处理 tensor = imageMat - .blobFromImageAndDoReleaseMat(1.0/255, mean, true) - .to4dFloatOnnxTensorAndNoReleaseMat(std,true); + .blobFromImageAndDoReleaseMat(1.0/255, this.mean, true) + .to4dFloatOnnxTensorAndNoReleaseMat(this.std,true); + //ONNX推理 + output = getSession().run(Collections.singletonMap(getInputName(), tensor)); + float[][][][] result = (float[][][][]) output.get(0).getValue(); + //模型后处理 + List plateInfos = new ArrayList<>(); + for (float[][] item : result[0]){ + List list = this.dbPostProcess(item, image.getWidth(), image.getHeight()); + if(!list.isEmpty()){ + plateInfos.addAll(list); + } + } + //返回 + return plateInfos; + }catch (Exception e){ + throw new RuntimeException(e); + }finally { + if(null != tensor){ + ReleaseUtil.release(tensor); + } + if(null != output){ + ReleaseUtil.release(output); + } + if(null != imageMat){ + ReleaseUtil.release(imageMat); + } + } + } + /** + * 后处理 + * @param pred + * @param destWidth + * @param destHeight + * @return + */ + private List dbPostProcess(float[][] pred, int destWidth, int destHeight){ + //初始化Mat + Mat rawMat = Mat.zeros(pred.length,pred[0].length, CvType.CV_8UC1); + Mat maskMat = Mat.zeros(pred.length,pred[0].length, CvType.CV_8UC1); + int width = rawMat.width(); int height = rawMat.height(); + //添加数据 + for(int i=0; i< pred.length; i++){ + for(int j=0; j= thresh ? 255 : 0); + } + } + //边缘提取 + final List points = new ArrayList<>(); + final Mat hierarchy = new Mat(); + Imgproc.findContours(maskMat, points, hierarchy, Imgproc.RETR_LIST, Imgproc.CHAIN_APPROX_SIMPLE); + //遍历边缘 + List list = new ArrayList<>(); + int numContours = Math.min(points.size(), max_candidates); + for(int i=0; i Double.valueOf(Math.floor(item.x)).intValue()).min().getAsInt(), 0, w-1); + int xmax = clip(Arrays.stream(box).mapToInt((item) -> Double.valueOf(Math.ceil(item.x)).intValue()).max().getAsInt(), 0, w-1); + int ymin = clip(Arrays.stream(box).mapToInt((item) -> Double.valueOf(Math.floor(item.y)).intValue()).min().getAsInt(), 0, h-1); + int ymax = clip(Arrays.stream(box).mapToInt((item) -> Double.valueOf(Math.ceil(item.y)).intValue()).max().getAsInt(), 0, h-1); + Mat mask = Mat.zeros(ymax - ymin + 1,xmax - xmin + 1, CvType.CV_8UC1); + //组织多边形的蒙版数据 + List boxPoints = Arrays.stream(box).map((item)-> new Point(item.x, item.y)).collect(Collectors.toList()); + List boxPointList = Arrays.stream(box).map((item)-> new Point(item.x-xmin, item.y-ymin)).collect(Collectors.toList()); + MatOfPoint matOfPoint = new MatOfPoint(); + matOfPoint.fromList(boxPointList); + List ptsoo = Collections.singletonList(matOfPoint); + //数据填充 + Imgproc.fillPoly(mask, ptsoo, Scalar.all(1)); + Scalar scalar = Core.mean(new Mat(rawMat, new Rect( xmin, ymin, xmax + 1 -xmin, ymax + 1-ymin)), mask); + float score = (float) scalar.val[0]; + if(this.box_thresh > score){ + continue; + } + //多边形的距离计算 + Polygon polygon = new Polygon(boxPoints); + double distance = polygon.getArea() * unclip_ratio / polygon.getLength(); + //线和多边形的裁剪和偏移 + Path path = new Path(); + for(Point u : boxPoints){ + path.add(new com.visual.open.anpr.core.clipper.Point.LongPoint((long) u.x, (long)u.y)); + } + Paths paths = new Paths(); + ClipperOffset offset = new ClipperOffset(); + offset.addPath(path, Clipper.JoinType.ROUND, Clipper.EndType.CLOSED_POLYGON); + offset.execute(paths, distance); + //重新获取边缘线 + MatOfPoint contour1 = new MatOfPoint(); + List poo= paths.get(0).stream().map(item -> new Point(item.getX(), item.getY())).collect(Collectors.toList()); + contour1.fromList(poo); + //重新提取边缘线 + PlateInfo res1 = getMiniBoxes(contour1); + if(res1.score < this.min_size+2){ + continue; + } + //添加一点偏移量 + int offsetWidth = (int) (res1.box.width() * 0.00f); + int offsetHeight = (int) (res1.box.height() * 0.00f); + //组装返回信息 + PlateInfo r = PlateInfo.build(score, PlateInfo.PlateBox.build( + PlateInfo.Point.build( + clip(Math.round(res1.box.leftTop.x / width * destWidth) - offsetWidth, 0, destWidth), + clip(Math.round(res1.box.leftTop.y / height * destHeight) - offsetHeight, 0, destHeight)), + PlateInfo.Point.build( + clip(Math.round(res1.box.rightTop.x / width * destWidth) + offsetWidth, 0, destWidth), + clip(Math.round(res1.box.rightTop.y / height * destHeight) - offsetHeight, 0, destHeight)), + PlateInfo.Point.build( + clip(Math.round(res1.box.rightBottom.x / width * destWidth) + offsetWidth, 0, destWidth), + clip(Math.round(res1.box.rightBottom.y / height * destHeight) + offsetHeight, 0, destHeight)), + PlateInfo.Point.build( + clip(Math.round(res1.box.leftBottom.x / width * destWidth) - offsetWidth, 0, destWidth), + clip(Math.round(res1.box.leftBottom.y / height * destHeight) + offsetHeight, 0, destHeight)) + )); + list.add(r); + } + return list; + } - }catch (Exception e){ - }finally { - ReleaseUtil.release(tensor); + public PlateInfo getMiniBoxes(MatOfPoint contour){ + MatOfPoint2f pts = new MatOfPoint2f(); + pts.fromList(contour.toList()); + RotatedRect rect = Imgproc.minAreaRect(pts); + + Mat yyy = new Mat(); + Imgproc.boxPoints(rect, yyy); + + List cs = new ArrayList<>(); + for (int m=0; m< yyy.size().height; m++){ + Point ccc = new Point(yyy.get(m, 0)[0], yyy.get(m, 1)[0]); + cs.add(ccc); + } + Collections.sort(cs, Comparator.comparingDouble(o -> o.x)); + + int index_1 = 0; + int index_2 = 1; + int index_3 = 2; + int index_4 = 3; + + if(cs.get(1).y > cs.get(0).y){ + index_1 = 0; + index_4 = 1; + }else{ + index_1 = 1; + index_4 = 0; } - return null; + if(cs.get(3).y > cs.get(2).y){ + index_2 = 2; + index_3 = 3; + }else{ + index_2 = 3; + index_3 = 2; + } + + double sside = Math.min(rect.size.height, rect.size.width); + PlateInfo.PlateBox plateBox = PlateInfo.PlateBox.build( + PlateInfo.Point.build((float) cs.get(index_1).x, (float)cs.get(index_1).y), + PlateInfo.Point.build((float) cs.get(index_2).x, (float)cs.get(index_2).y), + PlateInfo.Point.build((float) cs.get(index_3).x, (float)cs.get(index_3).y), + PlateInfo.Point.build((float) cs.get(index_4).x, (float)cs.get(index_4).y) + ); + return PlateInfo.build((float)sside, plateBox); } + public static int clip(int value, int min, int max){ + if(value > max){ + return max; + } + if(value < min){ + return min; + } + return value; + } } diff --git a/open-anpr-core/src/test/java/com/visual/open/anpr/core/models/PaddlePlateDetectionTest.java b/open-anpr-core/src/test/java/com/visual/open/anpr/core/models/PaddlePlateDetectionTest.java new file mode 100644 index 0000000..e5fcbb0 --- /dev/null +++ b/open-anpr-core/src/test/java/com/visual/open/anpr/core/models/PaddlePlateDetectionTest.java @@ -0,0 +1,41 @@ +package com.visual.open.anpr.core.models; + +import com.visual.open.anpr.core.domain.DrawImage; +import com.visual.open.anpr.core.domain.ImageMat; +import com.visual.open.anpr.core.domain.PlateInfo; + +import java.awt.*; +import java.util.HashMap; +import java.util.List; + +public class PaddlePlateDetectionTest { + + public static void main(String[] args) { + PaddlePlateDetection detection = new PaddlePlateDetection("open-anpr-core/src/main/resources/models/det.onnx", 1); + + String imagePath = "open-anpr-core/src/test/resources/images/image005.jpg"; + ImageMat imageMat = ImageMat.fromImage(imagePath); + List plateInfos = detection.inference(imageMat, 1,1, new HashMap<>()); + + DrawImage drawImage = DrawImage.build(imagePath); + for(PlateInfo plateInfo : plateInfos){ + PlateInfo.Point [] points = plateInfo.box.toArray(); + for(int i =0; i< points.length; i++){ + if(i+1 == points.length){ + drawImage.drawLine( + new DrawImage.Point((int)points[i].x, (int)points[i].y), + new DrawImage.Point((int)points[0].x, (int)points[0].y), + 2, Color.RED + ); + }else{ + drawImage.drawLine( + new DrawImage.Point((int)points[i].x, (int)points[i].y), + new DrawImage.Point((int)points[i+1].x, (int)points[i+1].y), + 2, Color.RED + ); + } + } + } + ImageMat.fromCVMat(drawImage.toMat()).imShow(); + } +} diff --git a/open-anpr-core/src/test/resources/images/image001.jpg b/open-anpr-core/src/test/resources/images/image001.jpg new file mode 100644 index 0000000..972f10e Binary files /dev/null and b/open-anpr-core/src/test/resources/images/image001.jpg differ diff --git a/open-anpr-core/src/test/resources/images/image002.jpg b/open-anpr-core/src/test/resources/images/image002.jpg new file mode 100644 index 0000000..215c8f6 Binary files /dev/null and b/open-anpr-core/src/test/resources/images/image002.jpg differ diff --git a/open-anpr-core/src/test/resources/images/image003.jpg b/open-anpr-core/src/test/resources/images/image003.jpg new file mode 100644 index 0000000..c80eb40 Binary files /dev/null and b/open-anpr-core/src/test/resources/images/image003.jpg differ diff --git a/open-anpr-core/src/test/resources/images/image004.jpg b/open-anpr-core/src/test/resources/images/image004.jpg new file mode 100644 index 0000000..eee6cac Binary files /dev/null and b/open-anpr-core/src/test/resources/images/image004.jpg differ diff --git a/open-anpr-core/src/test/resources/images/image005.jpg b/open-anpr-core/src/test/resources/images/image005.jpg new file mode 100644 index 0000000..2683bb0 Binary files /dev/null and b/open-anpr-core/src/test/resources/images/image005.jpg differ