class ZXLib::Basic::Tokenizer

# File lib/zxlib/basic.rb, line 702
def initialize(text, line_index=0, line_offset=0)
        @source = text
        @index = line_offset.to_i
        @token = nil
        @line_index = line_index
end

# File lib/zxlib/basic.rb, line 723
def next_token
        token = peek_token
        unless token.terminator?
                @token = nil
                token
        end
end

# File lib/zxlib/basic.rb, line 711
def parse_each(&block)
        enu = ::Enumerator.new do |y|
                while token = next_token
                        y << token
                end
        end
        if block_given?
                enu.each(&block)
        else
                enu
        end
end

# File lib/zxlib/basic.rb, line 730
def peek_token
        if @token.nil?
                if @source.empty?
                        @token = Token.new @index, nil, "", nil
                elsif m = Patterns::ESCAPE_MATCH.match(@source)
                        escstr = m.to_s
                        escexpr = m[1]
                        offset = m.end 0
                        @source = m.post_match
                        if m = Patterns::COLOR_CTRL_MATCH_EXACT.match(escexpr)
                                ctrl = CTRL_CODES[m[1]]
                                val = m[2].to_i
                                unless (0..31) === val
                                        raise SyntaxError, "special control arguments must be in a 0..31 range: #{escstr} in line: #{@line_index} at: #{@index}"
                                end
                                chars = [ctrl, val].pack('CC')
                                @token = Token.new @index, chars, chars, nil
                        elsif m = Patterns::CURSOR_CTRL_MATCH_EXACT.match(escexpr)
                                ctrl = CTRL_CODES[m[1]]
                                y, x = m[2].to_i, m[3]
                                if x.nil?
                                        chars = [ctrl, y].pack('Cv')
                                else
                                        chars = [ctrl, y, x.to_i].pack('CCC')
                                end
                                @token = Token.new @index, chars, chars, nil
                        elsif m = Patterns::KEYWORDS_MATCH_EXACT.match(escexpr)
                                code = KEYWORD_CODES[m.to_s]
                                @token = Token.new @index, m.to_s, [code].pack('C'), code
                        else
                                chars = ''
                                code = nil
                                while m = Patterns::ESCAPE_TOKEN_MATCH.match(escexpr)
                                        tok = m[1]
                                        unless code = NON_ASCII_ESCAPE_TOKENS[tok]
                                                code = Integer(tok)
                                                unless (0..255) === code
                                                        raise SyntaxError, "a code must be in a 0..255 range: #{escstr} in line: #{@line_index} at: #{@index}"
                                                end
                                        end
                                        chars << [code].pack('C')
                                        escexpr = m.post_match
                                end
                                unless escexpr.empty? && !chars.empty?
                                        raise SyntaxError, "unknown escape expression: #{escstr} in line: #{@line_index} at: #{@index}"
                                end
                                keyword_code = if chars.bytesize == 1 && (code >= KEYWORD_START_CODE || code < SE_NEW_KEYWORDS_END)
                                        code
                                end
                                @token = Token.new @index, chars, chars, keyword_code
                        end
                        @index += offset
                elsif m = Patterns::BINARY_EXPR_MATCH.match(@source)
                        key = m.to_s
                        @token = Token.new @index, key, key, KEYWORD_CODES['BIN']
                        @index += m.end 0
                        @source = m.post_match
                elsif m = Patterns::KEYWORDS_MATCH.match(@source)
                        key = m.to_s
                        @token = Token.new @index, key, key, KEYWORD_CODES[key.strip]
                        @index += m.end 0
                        @source = m.post_match
                elsif m = Patterns::NUMBER_MATCH.match(@source)
                        @token = Token.new @index, m.to_s, m.to_s, nil
                        @index += m.end 0
                        @source = m.post_match
                else
                        src = @source.slice!(0)
                        code = CHAR_CODES[src]
                        if code.nil?
                                raise SyntaxError, "not sure what to do with the character: \\u#{src.ord.to_s(16).rjust(4,?0)} in line: #{@line_index} at: #{@index}"
                        end
                        @token = Token.new @index, src, [code].pack('C'), nil
                        @index += 1
                end
        end
        @token
end

# File lib/zxlib/basic.rb, line 708
def terminated?
        peek_token.terminator?
end