【深度学习】基于套索的图像标注工具开发

参考文献

Reference：
[1] Mortensen E N, Barrett W A. Intelligent scissors for image composition[C]//Proceedings of the 22nd annual conference on Computer graphics and interactive techniques. ACM, 1995: 191-198.
[2] Mortensen E N, Barrett W A. Interactive segmentation with intelligent scissors[J]. Graphical models and image processing, 1998, 60(5): 349-384.
[3] CS 4670/5670, Project 1: Image Scissors
[4] Robert Sedgewick Kevin Wayne算法(中文第四版)

开源代码参考学习：
Opencv3实现智能剪刀算法 - IntelligentScissor
https://blog.youkuaiyun.com/DdogYuan/article/details/80554873

考虑之前的工作内容，需要将图片进行预处理，对龋齿进行标注，大多数场景下需要进行再调整，因此开源的标注工具Labelme不能满足工作需要。

使用C# winform进行桌面开发。

一、先上才艺

• 1、启动程序会自动在D盘根目录创建一个ToothDentalRes文件夹，里边包含两个原始文件夹
  
  
  Oragin文件夹中放入要标注的图片，Edited是生成的标注信息，目前只支持json格式。
• 2、启动后，输入默认账号和密码，1&1，进入标注主界面
  
  
• 3、点击读取图片，会默认打开D:\ToothDentalRes\Origin文件夹，选中一个图片
  
• 4、选中右侧龋齿级别（因为此次是为了牙齿标注做的，可以修改为其他），然后打开多点标注
  
• 5、保存当前图片，在生成json的文件如下：
  -

{
  "flag": {},
  "shapes": [
    {
      "label": "tooth",
      "line_color": null,
      "fill_color": null,
      "points": [
        [
          948,
          797
        ],
        [
          910,
          798
        ],
        [
          905,
          833
        ],
        [
          906,
          877
        ],
        [
          910,
          907
        ],
        [
          923,
          954
        ],
        [
          929,
          996
        ],
        [
          947,
          1037
        ],
        [
          971,
          1083
        ],
        [
          1016,
          1105
        ],
        [
          1057,
          1078
        ],
        [
          1094,
          1035
        ],
        [
          1132,
          977
        ],
        [
          1131,
          951
        ],
        [
          1089,
          909
        ],
        [
          1068,
          875
        ],
        [
          1031,
          842
        ],
        [
          991,
          814
        ]
      ]
    },
    {
      "label": "D",
      "line_color": null,
      "fill_color": null,
      "points": [
        [
          1156,
          842
        ],
        [
          1143,
          886
        ],
        [
          1155,
          934
        ],
        [
          1177,
          932
        ],
        [
          1204,
          917
        ],
        [
          1217,
          882
        ],
        [
          1207,
          852
        ],
        [
          1182,
          838
        ]
      ]
    },
    {
      "label": "B",
      "line_color": null,
      "fill_color": null,
      "points": [
        [
          968,
          1004
        ],
        [
          999,
          1002
        ],
        [
          1035,
          1015
        ],
        [
          1051,
          918
        ],
        [
          1045,
          893
        ],
        [
          966,
          839
        ],
        [
          947,
          834
        ],
        [
          922,
          853
        ],
        [
          938,
          910
        ],
        [
          944,
          961
        ],
        [
          927,
          992
        ]
      ]
    }
  ],
  "lineColor": [
    0,
    255,
    0,
    128
  ],
  "fillColor": [
    255,
    0,
    0,
    128
  ],
  "imagePath": "D:\\ToothDentalRes\\Edited\\龋齿图-1.jpg",
  "imageData": ""
}

二、实现思路

核心还是需要套索算法来进行实现，两种标注方式，一种是完全自由的，类似于画图板，只需要将所有的坐标保存即可。另一种是可调整的节点，我这里的处理是将节点作为一个方形图形加入，根据左上角坐标为点，进行所有节点连线，在移动的过程中进行重绘。

基础框架：Microsoft .NET Framework 4.6

三、核心代码

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace QsrTool.ScissorTool
{
    public class Vertex
    {
        public Tuple<int, int> Parent;
        public float Distance;
        public int ImageVerticesToParent; // Number of ImageVertices between current vertex and its parent
        public int i, j; // (i,j)->position of the vertex in the array ImageVertices
        public int index; // index of the vertex in the heap

        public Vertex(int x, int y)
        {
            j = y;
            i = x;
            Parent = null;
            Distance = float.MaxValue;
            ImageVerticesToParent = 0;
        }
    }

    public class priority_queue
    {
        int length;
        public void Swap<T>(ref T lhs, ref T rhs)
        {
            T temp;
            temp = lhs;
            lhs = rhs;
            rhs = temp;
        }
        public void Swap<T>(T lhs, T rhs)
        {
            T temp;
            temp = lhs;
            lhs = rhs;
            rhs = temp;
        }
        public int heap_size;
        public Vertex[] arr;
        public int parent(int i)
        {
            return i / 2;
        }
        public int left(int i)
        {
            return 2 * i;
        }
        public int right(int i)
        {
            return 2 * i + 1;
        }
        public void min_heapify(int i)
        {
            if (heap_size == 0)
                return;
            int l = left(i), r = right(i), smallest = i;
            if (l <= heap_size && arr[l].Distance < arr[smallest].Distance)
                smallest = l;
            if (r <= heap_size && arr[r].Distance < arr[smallest].Distance)
                smallest = r;
            if (smallest != i)
            {
                Swap<int>(ref arr[smallest].index, ref arr[i].index);
                Swap<Vertex>(ref arr[smallest], ref arr[i]);

                min_heapify(smallest);
            }

        }
        public priority_queue(ref Vertex[,] ImageVertices, int Height, int Width)
        {

            length = Height * Width;
            heap_size = length;
            arr = new Vertex[length + 1];
            int index = 1;
            for (int i = 0; i < Width; i++)
            {
                for (int j = 0; j < Height; j++)
                {
                    arr[index] = ImageVertices[i, j];
                    //ImageVertices[i, j].Distance = 757;
                    ImageVertices[i, j].index = index;
                    index++;
                }
            }

            for (int i = length / 2; i > 0; i--)
            {
                min_heapify(i);
            }
        }
        public Vertex extract_min()
        {
            Vertex min = arr[1];
            arr[1] = arr[heap_size];
            arr[1].index = 1;
            heap_size--;
            min_heapify(1);
            return min;
        }
    }

    public class Graph
    {
        //first dimension = right neighbour, Second = bottom, third = left, fourth = upper
        //public double[,,] Weight;
        public int Height, Width;
        RGBPixel[,] ImageMatrix;
        //First dimension represents the width and the second represents the height
        public Vertex[,] ImageVertices;
        public bool[,] isValid;
        public priority_queue Q;

        public Graph(RGBPixel[,] ImageMatrix)
        {
            this.ImageMatrix = ImageMatrix;
            //Get Width and Height
            Height = ImageOperations.GetHeight(ImageMatrix);
            Width = ImageOperations.GetWidth(ImageMatrix);
            GC.Collect();
            //Allocate 2D array of ImageVertices
            ImageVertices = new Vertex[Width, Height];
            isValid = new bool[Width, Height];
            for (int i = 0; i < Width; i++)
            {
                for (int j = 0; j < Height; j++)
                    isValid[i, j] = true;
            }
        }

        private float GetWeight(int x1, int y1, int x2, int y2)
        {

            if (x1 < x2) // Second cell to the right of the first cell
            {
                float temp = (float)ImageOperations.CalculatePixelEnergies(x1, y1, ImageMatrix).X;
                if (temp != 0) return (float)1.0 / temp;
                return 1E30f; // To avoid dividing by zero
            }
            else if (x2 < x1) // First cell to the right of the second cell
            {
                float temp = (float)ImageOperations.CalculatePixelEnergies(x2, y2, ImageMatrix).X;
                if (temp != 0) return (float)1.0 / temp;
                return 1E30f;
            }
            else if (y1 < y2) // Second cell below the first cell
            {
                float temp = (float)ImageOperations.CalculatePixelEnergies(x1, y1, ImageMatrix).Y;
                if (temp != 0) return (float)1.0 / temp;
                return 1E30f;
            }
            else // First cell below the second cell
            {
                float temp = (float)ImageOperations.CalculatePixelEnergies(x2, y2, ImageMatrix).Y;
                if (temp != 0) return (float)1.0 / temp;
                return 1E30f;
            }
        }

        void Relax_All(ref Vertex u)
        {
            //Relaxes the edges between u and all its neighbours
            if (u.j < Height - 1)
                Relax(ref u, ref ImageVertices[u.i, u.j + 1], GetWeight(u.i, u.j, u.i, u.j + 1));

            if (u.i < Width - 1)
                Relax(ref u, ref ImageVertices[u.i + 1, u.j], GetWeight(u.i, u.j, u.i + 1, u.j));

            if (u.j > 0)
                Relax(ref u, ref ImageVertices[u.i, u.j - 1], GetWeight(u.i, u.j, u.i, u.j - 1));

            if (u.i > 0)
                Relax(ref u, ref ImageVertices[u.i - 1, u.j], GetWeight(u.i, u.j, u.i - 1, u.j));
        }

        void Relax(ref Vertex u, ref Vertex v, float w)
        {
            //Relaxes the edges between u and v

            if (u == null || v == null || (!isValid[v.i, v.j]))
                return;

            if (v.Distance > u.Distance + w)
            {
                v.Distance = u.Distance + w;
                Tuple<int, int> temp = new Tuple<int, int>(u.i, u.j);
                v.Parent = temp;
                v.ImageVerticesToParent = u.ImageVerticesToParent + 1;
                while (Q.parent(v.index) > 0 && Q.arr[v.index].Distance < Q.arr[Q.parent(v.index)].Distance)
                {

                    Q.Swap<Vertex>(ref Q.arr[v.index], ref Q.arr[Q.parent(v.index)]);
                    Q.Swap<int>(ref Q.arr[v.index].index, ref Q.arr[Q.parent(v.index)].index);
                }
            }
        }

        public void Dijkstra(int x, int y)
        {


            #region Pruning
            //Pruning distance
            int diff = 250;
            Vertex[,] WindowVertices;
            int Width2 = 0, Height2 = 0, x1, x2, y1, y2;

            x1 = Math.Max(x - diff, 0);
            x2 = Math.Min(Width, x + diff);
            y1 = Math.Max(y - diff, 0);
            y2 = Math.Min(Height, y + diff);
            Width2 = x2 - x1;
            Height2 = y2 - y1;
            WindowVertices = new Vertex[Width2, Height2];

            //Destroys any previous calculations and calculate the shortest path from the given point
            for (int i = x1; i < x2; i++)
            {
                for (int j = y1; j < y2; j++)
                {
                    ImageVertices[i, j] = new Vertex(i, j);
                }
            }
            //Set the source distance to zero
            ImageVertices[x, y].Distance = 0;

            int w1 = 0, h1 = 0;
            for (int i = x1; i < x2; i++)
            {
                h1 = 0;
                for (int j = y1; j < y2; j++)
                {
                    WindowVertices[w1, h1] = ImageVertices[i, j];
                    h1++;
                }
                w1++;
            }
            //Priority queue using heap containing all ImageVertices
            Q = new priority_queue(ref WindowVertices, Height2, Width2);
            //End of Pruning
            #endregion

            while (Q.heap_size > 0)
            {
                //Extract the vertex with minimum distance and relax its edges
                Vertex u = Q.extract_min();
                int i = u.i, j = u.j;
                Relax_All(ref u);
            }
            Q = null;
            GC.Collect(); // Manually calling the garbage collector
        }
    }
}

using System;
using System.Collections.Generic;
using System.Text;
using System.Drawing;
using System.Windows.Forms;
using System.Drawing.Imaging;

namespace QsrTool.ScissorTool
{
    /// <summary>
    /// Holds the pixel color in 3 byte values: red, green and blue
    /// </summary>
    public struct RGBPixel
    {
        public byte red, green, blue;
    }
    public struct RGBPixelD
    {
        public double red, green, blue;
    }

    /// <summary>
    /// Holds the edge energy between 
    ///     1. a pixel and its right one (X)
    ///     2. a pixel and its bottom one (Y)
    /// </summary>
    public struct Vector2D
    {
        public double X { get; set; }
        public double Y { get; set; }
    }

    /// <summary>
    /// Library of static functions that deal with images
    /// </summary>
    public class ImageOperations
    {
        /// <summary>
        /// Open an image and load it into 2D array of colors (size: Height x Width)
        /// </summary>
        /// <param name="ImagePath">Image file path</param>
        /// <returns>2D array of colors</returns>
        public static RGBPixel[,] OpenImage(string ImagePath)
        {
            Bitmap original_bm = new Bitmap(ImagePath);
            int Height = original_bm.Height;
            int Width = original_bm.Width;

            RGBPixel[,] Buffer = new RGBPixel[Height, Width];

            unsafe
            {
                BitmapData bmd = original_bm.LockBits(new Rectangle(0, 0, Width, Height), ImageLockMode.ReadWrite, original_bm.PixelFormat);
                int x, y;
                int nWidth = 0;
                bool Format32 = false;
                bool Format24 = false;
                bool Format8 = false;

                if (original_bm.PixelFormat == PixelFormat.Format24bppRgb)
                {
                    Format24 = true;
                    nWidth = Width * 3;
                }
                else if (original_bm.PixelFormat == PixelFormat.Format32bppArgb || original_bm.PixelFormat == PixelFormat.Format32bppRgb || original_bm.PixelFormat == PixelFormat.Format32bppPArgb)
                {
                    Format32 = true;
                    nWidth = Width * 4;
                }
                else if (original_bm.PixelFormat == PixelFormat.Format8bppIndexed)
                {
                    Format8 = true;
                    nWidth = Width;
                }
                int nOffset = bmd.Stride - nWidth;
                byte* p = (byte*)bmd.Scan0;
                for (y = 0; y < Height; y++)
                {
                    for (x = 0; x < Width; x++)
                    {
                        if (Format8)
                        {
                            Buffer[y, x].red = Buffer[y, x].green = Buffer[y, x].blue = p[0];
                            p++;
                        }
                        else
                        {
                            Buffer[y, x].red = p[0];
                            Buffer[y, x].green = p[1];
                            Buffer[y, x].blue = p[2];
                            if (Format24) p += 3;
                            else if (Format32) p += 4;
                        }
                    }
                    p += nOffset;
                }
                original_bm.UnlockBits(bmd);
            }

            return Buffer;
        }

        /// <summary>
        /// Get the height of the image 
        /// </summary>
        /// <param name="ImageMatrix">2D array that contains the image</param>
        /// <returns>Image Height</returns>
        public static int GetHeight(RGBPixel[,] ImageMatrix)
        {
            return ImageMatrix.GetLength(0);
        }

        /// <summary>
        /// Get the width of the image 
        /// </summary>
        /// <param name="ImageMatrix">2D array that contains the image</param>
        /// <returns>Image Width</returns>
        public static int GetWidth(RGBPixel[,] ImageMatrix)
        {
            return ImageMatrix.GetLength(1);
        }

        /// <summary>
        /// Calculate edge energy between
        ///     1. the given pixel and its right one (X)
        ///     2. the given pixel and its bottom one (Y)
        /// </summary>
        /// <param name="x">pixel x-coordinate</param>
        /// <param name="y">pixel y-coordinate</param>
        /// <param name="ImageMatrix">colored image matrix</param>
        /// <returns>edge energy with the right pixel (X) and with the bottom pixel (Y)</returns>
        public static Vector2D CalculatePixelEnergies(int x, int y, RGBPixel[,] ImageMatrix)
        {
            if (ImageMatrix == null) throw new Exception("image is not set!");

            Vector2D gradient = CalculateGradientAtPixel(x, y, ImageMatrix);

            double gradientMagnitude = Math.Sqrt(gradient.X * gradient.X + gradient.Y * gradient.Y);
            double edgeAngle = Math.Atan2(gradient.Y, gradient.X);
            double rotatedEdgeAngle = edgeAngle + Math.PI / 2.0;

            Vector2D energy = new Vector2D();
            energy.X = Math.Abs(gradientMagnitude * Math.Cos(rotatedEdgeAngle));
            energy.Y = Math.Abs(gradientMagnitude * Math.Sin(rotatedEdgeAngle));

            return energy;
        }

        /// <summary>
        /// Display the given image on the given PictureBox object
        /// </summary>
        /// <param name="ImageMatrix">2D array that contains the image</param>
        /// <param name="PicBox">PictureBox object to display the image on it</param>
        public static void DisplayImage(RGBPixel[,] ImageMatrix, PictureBox PicBox)
        {
            // Create Image:
            //==============
            int Height = ImageMatrix.GetLength(0);
            int Width = ImageMatrix.GetLength(1);

            Bitmap ImageBMP = new Bitmap(Width, Height, PixelFormat.Format24bppRgb);

            unsafe
            {
                BitmapData bmd = ImageBMP.LockBits(new Rectangle(0, 0, Width, Height), ImageLockMode.ReadWrite, ImageBMP.PixelFormat);
                int nWidth = 0;
                nWidth = Width * 3;
                int nOffset = bmd.Stride - nWidth;
                byte* p = (byte*)bmd.Scan0;
                for (int i = 0; i < Height; i++)
                {
                    for (int j = 0; j < Width; j++)
                    {
                        p[0] = ImageMatrix[i, j].red;
                        p[1] = ImageMatrix[i, j].green;
                        p[2] = ImageMatrix[i, j].blue;
                        p += 3;
                    }

                    p += nOffset;
                }
                ImageBMP.UnlockBits(bmd);
            }
            PicBox.Image = ImageBMP;
            GC.Collect();
        }


        /// <summary>
        /// Apply Gaussian smoothing filter to enhance the edge detection 
        /// </summary>
        /// <param name="ImageMatrix">Colored image matrix</param>
        /// <param name="filterSize">Gaussian mask size</param>
        /// <param name="sigma">Gaussian sigma</param>
        /// <returns>smoothed color image</returns>
        public static RGBPixel[,] GaussianFilter1D(RGBPixel[,] ImageMatrix, int filterSize, double sigma)
        {
            int Height = GetHeight(ImageMatrix);
            int Width = GetWidth(ImageMatrix);

            RGBPixelD[,] VerFiltered = new RGBPixelD[Height, Width];
            RGBPixel[,] Filtered = new RGBPixel[Height, Width];


            // Create Filter in Spatial Domain:
            //=================================
            //make the filter ODD size
            if (filterSize % 2 == 0) filterSize++;

            double[] Filter = new double[filterSize];

            //Compute Filter in Spatial Domain :
            //==================================
            double Sum1 = 0;
            int HalfSize = filterSize / 2;
            for (int y = -HalfSize; y <= HalfSize; y++)
            {
                //Filter[y+HalfSize] = (1.0 / (Math.Sqrt(2 * 22.0/7.0) * Segma)) * Math.Exp(-(double)(y*y) / (double)(2 * Segma * Segma)) ;
                Filter[y + HalfSize] = Math.Exp(-(double)(y * y) / (double)(2 * sigma * sigma));
                Sum1 += Filter[y + HalfSize];
            }
            for (int y = -HalfSize; y <= HalfSize; y++)
            {
                Filter[y + HalfSize] /= Sum1;
            }

            //Filter Original Image Vertically:
            //=================================
            int ii, jj;
            RGBPixelD Sum;
            RGBPixel Item1;
            RGBPixelD Item2;

            for (int j = 0; j < Width; j++)
                for (int i = 0; i < Height; i++)
                {
                    Sum.red = 0;
                    Sum.green = 0;
                    Sum.blue = 0;
                    for (int y = -HalfSize; y <= HalfSize; y++)
                    {
                        ii = i + y;
                        if (ii >= 0 && ii < Height)
                        {
                            Item1 = ImageMatrix[ii, j];
                            Sum.red += Filter[y + HalfSize] * Item1.red;
                            Sum.green += Filter[y + HalfSize] * Item1.green;
                            Sum.blue += Filter[y + HalfSize] * Item1.blue;
                        }
                    }
                    VerFiltered[i, j] = Sum;
                }

            //Filter Resulting Image Horizontally:
            //===================================
            for (int i = 0; i < Height; i++)
                for (int j = 0; j < Width; j++)
                {
                    Sum.red = 0;
                    Sum.green = 0;
                    Sum.blue = 0;
                    for (int x = -HalfSize; x <= HalfSize; x++)
                    {
                        jj = j + x;
                        if (jj >= 0 && jj < Width)
                        {
                            Item2 = VerFiltered[i, jj];
                            Sum.red += Filter[x + HalfSize] * Item2.red;
                            Sum.green += Filter[x + HalfSize] * Item2.green;
                            Sum.blue += Filter[x + HalfSize] * Item2.blue;
                        }
                    }
                    Filtered[i, j].red = (byte)Sum.red;
                    Filtered[i, j].green = (byte)Sum.green;
                    Filtered[i, j].blue = (byte)Sum.blue;
                }

            return Filtered;
        }


        #region Private Functions
        /// <summary>
        /// Calculate Gradient vector between the given pixel and its right and bottom ones
        /// </summary>
        /// <param name="x">pixel x-coordinate</param>
        /// <param name="y">pixel y-coordinate</param>
        /// <param name="ImageMatrix">colored image matrix</param>
        /// <returns></returns>
        private static Vector2D CalculateGradientAtPixel(int x, int y, RGBPixel[,] ImageMatrix)
        {
            Vector2D gradient = new Vector2D();

            RGBPixel mainPixel = ImageMatrix[y, x];
            double pixelGrayVal = 0.21 * mainPixel.red + 0.72 * mainPixel.green + 0.07 * mainPixel.blue;

            if (y == GetHeight(ImageMatrix) - 1)
            {
                //boundary pixel.
                for (int i = 0; i < 3; i++)
                {
                    gradient.Y = 0;
                }
            }
            else
            {
                RGBPixel downPixel = ImageMatrix[y + 1, x];
                double downPixelGrayVal = 0.21 * downPixel.red + 0.72 * downPixel.green + 0.07 * downPixel.blue;

                gradient.Y = pixelGrayVal - downPixelGrayVal;
            }

            if (x == GetWidth(ImageMatrix) - 1)
            {
                //boundary pixel.
                gradient.X = 0;

            }
            else
            {
                RGBPixel rightPixel = ImageMatrix[y, x + 1];
                double rightPixelGrayVal = 0.21 * rightPixel.red + 0.72 * rightPixel.green + 0.07 * rightPixel.blue;

                gradient.X = pixelGrayVal - rightPixelGrayVal;
            }


            return gradient;
        }

        internal static int GetHeight(object imageMatrix)
        {
            throw new NotImplementedException();
        }


        #endregion
    }
}

后续可优化

• 当前保存节点的形式是队列，有先后顺序，因此会出现节点交叉的情况。后期可以弱化节点的时间顺序，根据横纵坐标进行判断，重新排序
• 多种方式进行绘制并存
• 算法优化，等等大佬们发论文吧

release程序下载：
链接：https://pan.baidu.com/s/1vd-MRYDj5YjKR_rVA2QtSg
提取码：qiao