133 lines
2.1 KiB
C++
133 lines
2.1 KiB
C++
// License: Modified MIT (NON-AI)
|
|
// See the LICENSE file in the root directory for license information.
|
|
// Copyright 2025 Timothy Prepscius
|
|
|
|
#pragma once
|
|
|
|
#include "Tokenizer.h"
|
|
#include <string>
|
|
#include <vector>
|
|
#include "str_find.h"
|
|
|
|
namespace tjp {
|
|
namespace core {
|
|
|
|
struct TokenVector
|
|
{
|
|
typedef std::vector<unsigned char> Tokens;
|
|
Tokens tokens;
|
|
|
|
TokenVector(const std::string &_del) :
|
|
tokens(255, false)
|
|
{
|
|
const unsigned char *begin = (const unsigned char *)_del.c_str();
|
|
const unsigned char *end = begin + _del.size();
|
|
for (const unsigned char *c=begin; c!=end;++c)
|
|
tokens[*c] = true;
|
|
}
|
|
|
|
size_t find(const unsigned char *s, size_t left, char &t, size_t size) const
|
|
{
|
|
return str_find(s, tokens.data(), left, t, size);
|
|
}
|
|
} ;
|
|
|
|
struct TokenStringView
|
|
{
|
|
typedef std::string_view Tokens;
|
|
Tokens tokens;
|
|
|
|
TokenStringView(const std::string_view &_del) :
|
|
tokens(_del)
|
|
{
|
|
}
|
|
|
|
size_t find(const unsigned char *s, size_t left, char &t, size_t size) const
|
|
{
|
|
return str_find(s, tokens, left, t, size);
|
|
}
|
|
} ;
|
|
|
|
template<typename string, typename token_type>
|
|
class Tokenizer
|
|
{
|
|
public:
|
|
typedef token_type Tokens;
|
|
typedef size_t index;
|
|
|
|
protected:
|
|
const unsigned char *s;
|
|
const Tokens *tokens;
|
|
|
|
index size;
|
|
index left;
|
|
index right;
|
|
char t;
|
|
|
|
public:
|
|
|
|
Tokenizer (const char *_s, index _size, const token_type &tokens_) :
|
|
s((const unsigned char *)_s),
|
|
left(0),
|
|
tokens(&tokens_),
|
|
size(_size)
|
|
{
|
|
right = tokens->find(s, left, t, size);
|
|
}
|
|
|
|
Tokenizer () :
|
|
s(nullptr),
|
|
tokens(nullptr),
|
|
left(0),
|
|
right(0),
|
|
size(0)
|
|
{
|
|
|
|
}
|
|
|
|
bool next (string &_s, char &_t)
|
|
{
|
|
index _left,_size;
|
|
|
|
bool result = next(_left, _size, _t);
|
|
if (result)
|
|
_s = string((const char *)s+_left, _size);
|
|
|
|
return result;
|
|
}
|
|
|
|
bool next (index &_left, index &_size, char &_t)
|
|
{
|
|
if (left != -1)
|
|
{
|
|
_t = t;
|
|
|
|
if (right == -1)
|
|
{
|
|
_left = left;
|
|
_size = size - left;
|
|
left = right;
|
|
}
|
|
else
|
|
{
|
|
_left = left;
|
|
_size = right-left;
|
|
|
|
left = right + 1;
|
|
if (left < size)
|
|
right = tokens->find(s, left, t, size);
|
|
else
|
|
left = -1;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
} ;
|
|
|
|
|
|
} // namespace
|
|
} // namespace
|