flatten 20260225

tjp/core/string/Tokenizer.hpp

@@ -0,0 +1,132 @@
+// License: Modified MIT (NON-AI)
+// See the LICENSE file in the root directory for license information.
+// Copyright 2025 Timothy Prepscius
+
+#pragma once
+
+#include "Tokenizer.h"
+#include <string>
+#include <vector>
+#include "str_find.h"
+
+namespace tjp {
+namespace core {
+
+struct TokenVector
+{
+	typedef std::vector<unsigned char> Tokens;
+	Tokens tokens;
+	
+	TokenVector(const std::string &_del) :
+		tokens(255, false)
+	{
+		const unsigned char *begin = (const unsigned char *)_del.c_str();
+		const unsigned char *end = begin + _del.size();
+		for (const unsigned char *c=begin; c!=end;++c)
+			tokens[*c] = true;
+	}
+
+	size_t find(const unsigned char *s, size_t left, char &t, size_t size) const
+	{
+		return str_find(s, tokens.data(), left, t, size);
+	}
+} ;
+
+struct TokenStringView
+{
+	typedef std::string_view Tokens;
+	Tokens tokens;
+	
+	TokenStringView(const std::string_view &_del) :
+		tokens(_del)
+	{
+	}
+
+	size_t find(const unsigned char *s, size_t left, char &t, size_t size) const
+	{
+		return str_find(s, tokens, left, t, size);
+	}
+} ;
+
+template<typename string, typename token_type>
+class Tokenizer
+{
+public:
+	typedef token_type Tokens;
+	typedef size_t index;
+
+protected:
+	const unsigned char *s;
+	const Tokens *tokens;
+	
+	index size;
+	index left;
+	index right;
+	char t;
+	
+public:
+	
+	Tokenizer (const char *_s, index _size, const token_type &tokens_) :
+		s((const unsigned char *)_s),
+		left(0),
+		tokens(&tokens_),
+		size(_size)
+	{
+		right = tokens->find(s, left, t, size);
+	}
+	
+	Tokenizer () :
+		s(nullptr),
+		tokens(nullptr),
+		left(0),
+		right(0),
+		size(0)
+	{
+	
+	}
+	
+	bool next (string &_s, char &_t)
+	{
+		index _left,_size;
+		
+		bool result = next(_left, _size, _t);
+		if (result)
+			_s = string((const char *)s+_left, _size);
+		
+		return result;
+	}
+	
+	bool next (index &_left, index &_size, char &_t)
+	{
+		if (left != -1)
+		{
+			_t = t;
+			
+			if (right == -1)
+			{
+				_left = left;
+				_size = size - left;
+				left = right;
+			}
+			else
+			{
+				_left = left;
+				_size = right-left;
+				
+				left = right + 1;
+				if (left < size)
+					right = tokens->find(s, left, t, size);
+				else
+					left = -1;
+			}
+			
+			return true;
+		}
+		
+		return false;
+	}
+} ;
+
+
+} // namespace
+} // namespace