token-stream.hpp

/* Copyright 2025 Jonathan S. Arney
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      https://github.com/jarney/gyoji/blob/master/LICENSE
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */
#pragma once
#include <string>
#include <vector>
#include <map>
#include <algorithm>
#include <gyoji-misc/pointers.hpp>
#include <gyoji-context/source-reference.hpp>
 
namespace Gyoji::context {
    class Token;
    class TokenStream;
 
    typedef unsigned int TokenID;
    
    class Token {
    public:
    Token(
        TokenID _typestr,
        std::string _value,
        const std::string & _filename,
        size_t _line,
        size_t _column,
        size_t _length
        );
    ~Token();
    const TokenID & get_type() const;
    const std::string & get_value() const;
 
    const SourceReference & get_source_ref() const;
    
    friend TokenStream;
    private:
    void append(std::string & value);
    TokenID typestr;
    std::string value;
    SourceReference src_ref;
    };
 
    class TokenStream {
    public:
    TokenStream();
    ~TokenStream();
    
    const std::vector<Gyoji::owned<Token>> & get_tokens() const;
    
    const SourceReference & get_current_source_ref() const;
    
    std::string get_line(size_t _line) const;
    
    const Token & add_token(
        TokenID _typestr,
        std::string _value,
        const std::string & _filename,
        size_t _line,
        size_t _column
        );
    
    std::vector<std::pair<size_t, std::string>> context(size_t line_start, size_t line_end) const;
    
    void append_token(std::string _value);
 
    static const SourceReference & get_zero_source_ref();
 
    private:
    
    std::vector<Gyoji::owned<Token>> tokens;
    std::map<size_t, std::vector<Token*>> tokens_by_lineno;
    };
};