19、Tokenizer

Tokenizer

该项目使用一个接口和类层次结构来模拟读取C#源文件和将源文件分类为各种标记（标识、关键字和操作符等）。作为一个用法示例，它还从关键的接口派生类以便以彩色语法的形式在一个富文本框内显示标记。

(1) 创建项目

创建WPF Application项目：Tokenizer。

(2) 添加资源

① 添加TokenInterface文件夹，并在其中添加Interface文件：IVisitable.cs、IToken.cs、ITokenVisitor.cs和IVisitableToken.cs。

② 添加TokenClass文件夹，并在其中添加Class文件：DefaultTokenImpl.cs、IdentifierToken.cs、KeywordToken.cs、OperatorToken.cs、PunctuatorToken.cs、StringLiteralToken.cs和WhitespaceToken.cs。

③ 添加Class文件：ColorSyntaxVisitor.cs和SourceFile.cs。

(3) 设计界面

 

<Window x:Class="Tokenizer.MainWindow"

        xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"

        xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"

        Title="Color Syntax" Height="300" Width="500">

    <Grid>

        <RichTextBox Name="richTextBoxCodeText" Margin="10,10,10,40" HorizontalScrollBarVisibility="Auto" VerticalScrollBarVisibility="Auto">

            <RichTextBox.Resources>

                <Style TargetType="{x:Type Paragraph}">

                    <Setter Property="Margin" Value="0"/>

                </Style>

            </RichTextBox.Resources>

        </RichTextBox>

        <Button Name="buttonOpen" Margin="0,0,0,10" Width="65" Height="23" HorizontalAlignment="Center" VerticalAlignment="Bottom" Content="Open" Click="buttonOpen_Click" />

    </Grid>

</Window>

(4) 修改IVisitable.cs

using System;

using System.Collections.Generic;

using System.Linq;

using System.Text;

 

namespace Tokenizer. TokenInterface

{

    interface IVisitable

    {

        void Accept(ITokenVisitor visitor);

    }

}

(5) 修改IToken.cs

using System;

using System.Collections.Generic;

using System.Linq;

using System.Text;

 

namespace Tokenizer. TokenInterface

{

    interface IToken

    {

        string ToString();

    }

}

(6) 修改ITokenVisitor.cs

using System;

using System.Collections.Generic;

using System.Linq;

using System.Text;

 

namespace Tokenizer.TokenInterface

{

    interface ITokenVisitor

    {

        void VisitComment(string token);

        void VisitIdentifier(string token);

        void VisitKeyword(string token);

        void VisitOperator(string token);

        void VisitPunctuator(string token);

        void VisitStringLiteral(string token);

        void VisitWhitespace(string token);

    }

}

(7) 修改IVisitableToken.cs

using System;

using System.Collections.Generic;

using System.Linq;

using System.Text;

 

namespace Tokenizer.TokenInterface

{

    interface IVisitableToken : IVisitable, IToken

    {

 

    }

}

(8) 修改DefaultTokenImpl.cs

using System;

using System.Collections.Generic;

using System.Linq;

using System.Text;

 

namespace Tokenizer.TokenClass

{

    abstract class DefaultTokenImpl

    {

        private readonly string name;

 

        protected DefaultTokenImpl(string name)

        {

            this.name = name;

        }

 

        public new string ToString()

        {

            return this.name;

        }

    }

}

(9) 修改IdentifierToken.cs

using System;

using System.Collections.Generic;

using System.Linq;

using System.Text;

using Tokenizer.TokenInterface;

 

namespace Tokenizer.TokenClass

{

    class IdentifierToken : DefaultTokenImpl, IVisitableToken

    {

        public IdentifierToken(string name)

            : base(name)

        {

        }

 

        void IVisitable.Accept(ITokenVisitor visitor)

        {

            visitor.VisitIdentifier(this.ToString());

        }

    }

}

(10) 修改KeywordToken.cs

using System;

using System.Collections.Generic;

using System.Linq;

using System.Text;

using Tokenizer.TokenInterface;

 

namespace Tokenizer.TokenClass

{

    class KeywordToken : DefaultTokenImpl, IVisitableToken

    {

        public KeywordToken(string name)

            : base(name)

        {

        }

 

        void IVisitable.Accept(ITokenVisitor visitor)

        {

            visitor.VisitKeyword(this.ToString());

        }

    }

}

(11) 修改OperatorToken.cs

using System;

using System.Collections.Generic;

using System.Linq;

using System.Text;

using Tokenizer.TokenInterface;

 

namespace Tokenizer.TokenClass

{

    class OperatorToken : DefaultTokenImpl, IVisitableToken

    {

        public OperatorToken(string name)

            : base(name)

        {

        }

 

        void IVisitable.Accept(ITokenVisitor visitor)

        {

            visitor.VisitOperator(this.ToString());

        }

    }

}

(12) 修改PunctuatorToken.cs

using System;

using System.Collections.Generic;

using System.Linq;

using System.Text;

using Tokenizer.TokenInterface;

 

namespace Tokenizer.TokenClass

{

    class PunctuatorToken : DefaultTokenImpl, IVisitableToken

    {

        public PunctuatorToken(string name)

            : base(name)

        {

        }

 

        void IVisitable.Accept(ITokenVisitor visitor)

        {

            visitor.VisitPunctuator(this.ToString());

        }

    }

}

(13) 修改StringLiteralToken.cs

using System;

using System.Collections.Generic;

using System.Linq;

using System.Text;

using Tokenizer.TokenInterface;

 

namespace Tokenizer.TokenClass

{

    class StringLiteralToken : DefaultTokenImpl, IVisitableToken

    {

        public StringLiteralToken(string name)

            : base(name)

        {

        }

 

        void IVisitable.Accept(ITokenVisitor visitor)

        {

            visitor.VisitStringLiteral(this.ToString());

        }

    }

}

(14) 修改WhitespaceToken.cs

using System;

using System.Collections.Generic;

using System.Linq;

using System.Text;

using Tokenizer.TokenInterface;

 

namespace Tokenizer.TokenClass

{

    class WhitespaceToken : DefaultTokenImpl, IVisitableToken

    {

        public WhitespaceToken(string name)

            : base(name)

        {

        }

 

        void IVisitable.Accept(ITokenVisitor visitor)

        {

            visitor.VisitWhitespace(this.ToString());

        }

    }

}

(15) 修改ColorSyntaxVisitor.cs

using System;

using System.Collections.Generic;

using System.Linq;

using System.Text;

using Tokenizer.TokenInterface;

using System.Windows.Controls;

using System.Windows.Media;

using System.Windows.Documents;

 

namespace Tokenizer

{

    sealed class ColorSyntaxVisitor : ITokenVisitor

    {

        private readonly RichTextBox target;

 

        public ColorSyntaxVisitor(RichTextBox rtb)

        {

            this.target = rtb;

            this.target.Document.Blocks.Clear();

        }

 

        private void Write(string token, SolidColorBrush color)

        {

            target.AppendText(token);

            int offsetToStartOfToken = -1 * token.Length - 2;

            int offsetToEndOfToken = -2;

            TextPointer start =

                target.Document.ContentEnd.GetPositionAtOffset(offsetToStartOfToken);

            TextPointer end =

                target.Document.ContentEnd.GetPositionAtOffset(offsetToEndOfToken);

            TextRange text = new TextRange(start, end);

            text.ApplyPropertyValue(TextElement.ForegroundProperty, color);

        }

 

        #region ITokenVisitor Members

        void ITokenVisitor.VisitComment(string token)

        {

            Write(token, Brushes.Black);

        }

 

        void ITokenVisitor.VisitIdentifier(string token)

        {

            Write(token, Brushes.Black);

        }

 

        void ITokenVisitor.VisitKeyword(string token)

        {

            Write(token, Brushes.Blue);

        }

 

        void ITokenVisitor.VisitOperator(string token)

        {

            Write(token, Brushes.Black);

        }

 

        void ITokenVisitor.VisitPunctuator(string token)

        {

            Write(token, Brushes.Black);

        }

 

        void ITokenVisitor.VisitStringLiteral(string token)

        {

            Write(token, Brushes.Green);

        }

 

        void ITokenVisitor.VisitWhitespace(string token)

        {

            Write(token, Brushes.Black);

        }

        #endregion

    }

}

(16) 修改SourceFile.cs

using System;

using System.Collections.Generic;

using System.Linq;

using System.Text;

using Tokenizer.TokenInterface;

using Tokenizer.TokenClass;

 

namespace Tokenizer

{

    class SourceFile : IVisitable

    {

        private IVisitableToken[] tokens =

              {

                     new KeywordToken("using"),

                     new WhitespaceToken(" "),

                     new IdentifierToken("System"),

                     new PunctuatorToken(";"),

                     new WhitespaceToken("/n"),

                     new WhitespaceToken("/n"),

                     new KeywordToken("class"),

                     new WhitespaceToken(" "),

                     new IdentifierToken("Greeting"),

                     new WhitespaceToken("/n"),

                     new PunctuatorToken("{"),

                     new WhitespaceToken("/n"),

                     new WhitespaceToken("    "),

                     new KeywordToken("static"),

                     new WhitespaceToken(" "),

                     new KeywordToken("void"),

                     new WhitespaceToken(" "),

                     new IdentifierToken("Main"),

                     new PunctuatorToken("("),

                     new PunctuatorToken(")"),

                     new WhitespaceToken("/n"),

                     new WhitespaceToken("    "),

                     new PunctuatorToken("{"),

                     new WhitespaceToken("/n"),

                     new WhitespaceToken("        "),

                     new IdentifierToken("Console"),

                     new OperatorToken("."),

                     new IdentifierToken("WriteLine"),

                     new PunctuatorToken("("),

                     new StringLiteralToken("/"Hello, world/""),

                     new PunctuatorToken(")"),

                     new PunctuatorToken(";"),

                     new WhitespaceToken("/n"),

                     new WhitespaceToken("    "),

                     new PunctuatorToken("}"),

                     new WhitespaceToken("/n"),

                     new PunctuatorToken("}"),

              };

 

        public void Accept(ITokenVisitor visitor)

        {

            foreach (IVisitableToken token in tokens)

            {

                token.Accept(visitor);

            }

        }

    }

}

(17) 修改MainWindow.xaml.cs

using System;

using System.Collections.Generic;

using System.Linq;

using System.Text;

using System.Windows;

using System.Windows.Controls;

using System.Windows.Data;

using System.Windows.Documents;

using System.Windows.Input;

using System.Windows.Media;

using System.Windows.Media.Imaging;

using System.Windows.Navigation;

using System.Windows.Shapes;

 

namespace Tokenizer

{

    /// <summary>

    /// Interaction logic for MainWindow.xaml

    /// </summary>

    public partial class MainWindow : Window

    {

        public MainWindow()

        {

            InitializeComponent();

        }

 

        private void buttonOpen_Click(object sender, RoutedEventArgs e)

        {

            SourceFile source = new SourceFile();

            ColorSyntaxVisitor visitor = new ColorSyntaxVisitor(richTextBoxCodeText);

            source.Accept(visitor);

        }

    }

}

(18) 运行测试

 

资源下载