// License: Modified MIT (NON-AI)
// See the LICENSE file in the root directory for license information.
// Copyright 2025 Timothy Prepscius

#pragma once

#include "Tokenizer.h"
#include <string>
#include <vector>
#include "str_find.h"

namespace tjp {
namespace core {

struct TokenVector
{
	typedef std::vector<unsigned char> Tokens;
	Tokens tokens;
	
	TokenVector(const std::string &_del) :
		tokens(255, false)
	{
		const unsigned char *begin = (const unsigned char *)_del.c_str();
		const unsigned char *end = begin + _del.size();
		for (const unsigned char *c=begin; c!=end;++c)
			tokens[*c] = true;
	}

	size_t find(const unsigned char *s, size_t left, char &t, size_t size) const
	{
		return str_find(s, tokens.data(), left, t, size);
	}
} ;

struct TokenStringView
{
	typedef std::string_view Tokens;
	Tokens tokens;
	
	TokenStringView(const std::string_view &_del) :
		tokens(_del)
	{
	}

	size_t find(const unsigned char *s, size_t left, char &t, size_t size) const
	{
		return str_find(s, tokens, left, t, size);
	}
} ;

template<typename string, typename token_type>
class Tokenizer
{
public:
	typedef token_type Tokens;
	typedef size_t index;

protected:
	const unsigned char *s;
	const Tokens *tokens;
	
	index size;
	index left;
	index right;
	char t;
	
public:
	
	Tokenizer (const char *_s, index _size, const token_type &tokens_) :
		s((const unsigned char *)_s),
		left(0),
		tokens(&tokens_),
		size(_size)
	{
		right = tokens->find(s, left, t, size);
	}
	
	Tokenizer () :
		s(nullptr),
		tokens(nullptr),
		left(0),
		right(0),
		size(0)
	{
	
	}
	
	bool next (string &_s, char &_t)
	{
		index _left,_size;
		
		bool result = next(_left, _size, _t);
		if (result)
			_s = string((const char *)s+_left, _size);
		
		return result;
	}
	
	bool next (index &_left, index &_size, char &_t)
	{
		if (left != -1)
		{
			_t = t;
			
			if (right == -1)
			{
				_left = left;
				_size = size - left;
				left = right;
			}
			else
			{
				_left = left;
				_size = right-left;
				
				left = right + 1;
				if (left < size)
					right = tokens->find(s, left, t, size);
				else
					left = -1;
			}
			
			return true;
		}
		
		return false;
	}
} ;


} // namespace
} // namespace