token-stream.hpp

#pragma once
#include <string>
#include <vector>
#include <map>
#include <algorithm>
#include <gyoji-misc/pointers.hpp>
#include <gyoji-context/source-reference.hpp>
 
namespace Gyoji::context {
    class Token;
    class TokenStream;
 
    typedef unsigned int TokenID;
    
    class Token {
    public:
    Token(
        TokenID _typestr,
        std::string _value,
        const std::string & _filename,
        size_t _line,
        size_t _column,
        size_t _length
        );
    ~Token();
    const TokenID & get_type() const;
    const std::string & get_value() const;
 
    const SourceReference & get_source_ref() const;
    
    friend TokenStream;
    private:
    void append(std::string & value);
    TokenID typestr;
    std::string value;
    SourceReference src_ref;
    };
 
    class TokenStream {
    public:
    TokenStream();
    ~TokenStream();
    
    const std::vector<Gyoji::owned<Token>> & get_tokens() const;
    
    const SourceReference & get_current_source_ref() const;
    
    std::string get_line(size_t _line) const;
    
    const Token & add_token(
        TokenID _typestr,
        std::string _value,
        const std::string & _filename,
        size_t _line,
        size_t _column
        );
    
    std::vector<std::pair<size_t, std::string>> context(size_t line_start, size_t line_end) const;
    
    void append_token(std::string _value);
 
    static const SourceReference & get_zero_source_ref();
 
    private:
    
    std::vector<Gyoji::owned<Token>> tokens;
    std::map<size_t, std::vector<Token*>> tokens_by_lineno;
    };
};