tprepscius/core_misc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

flatten 20260225

Browse Source
This commit is contained in:
Timothy Prepscius
2026-02-25 12:39:24 -05:00
commit fa54be052a
315 changed files with 49791 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored Normal file
View File

@@ -0,0 +1,4 @@
.DS_Store
*.pyc
xcuserdata
.bin

1753
Core_Misc.xcodeproj/project.pbxproj Normal file
View File

File diff suppressed because it is too large Load Diff

4
Core_Misc.xcodeproj/project.xcworkspace/contents.xcworkspacedata generated Normal file
View File

@@ -0,0 +1,4 @@
<?xml version="1.0" encoding="UTF-8"?>
<Workspace
version = "1.0">
</Workspace>

8
Core_Misc.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist Normal file
View File

@@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>IDEDidComputeMac32BitWarning</key>
<true/>
</dict>
</plist>

146
Core_Misc.xcodeproj/xcshareddata/xcschemes/Core_Misc_Tests.xcscheme Normal file
View File

@@ -0,0 +1,146 @@
<?xml version="1.0" encoding="UTF-8"?>
<Scheme
LastUpgradeVersion = "1330"
version = "1.3">
<BuildAction
parallelizeBuildables = "YES"
buildImplicitDependencies = "YES">
<BuildActionEntries>
<BuildActionEntry
buildForTesting = "YES"
buildForRunning = "YES"
buildForProfiling = "YES"
buildForArchiving = "YES"
buildForAnalyzing = "YES">
<BuildableReference
BuildableIdentifier = "primary"
BlueprintIdentifier = "F608AA9728270442005C276B"
BuildableName = "Core_Misc_Tests"
BlueprintName = "Core_Misc_Tests"
ReferencedContainer = "container:Core_Misc.xcodeproj">
</BuildableReference>
</BuildActionEntry>
</BuildActionEntries>
</BuildAction>
<TestAction
buildConfiguration = "Debug"
selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
shouldUseLaunchSchemeArgsEnv = "YES">
<Testables>
</Testables>
</TestAction>
<LaunchAction
buildConfiguration = "Debug"
selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
enableAddressSanitizer = "YES"
enableASanStackUseAfterReturn = "YES"
launchStyle = "0"
useCustomWorkingDirectory = "NO"
ignoresPersistentStateOnLaunch = "NO"
debugDocumentVersioning = "YES"
debugServiceExtension = "internal"
allowLocationSimulation = "YES">
<BuildableProductRunnable
runnableDebuggingMode = "0">
<BuildableReference
BuildableIdentifier = "primary"
BlueprintIdentifier = "F608AA9728270442005C276B"
BuildableName = "Core_Misc_Tests"
BlueprintName = "Core_Misc_Tests"
ReferencedContainer = "container:Core_Misc.xcodeproj">
</BuildableReference>
</BuildableProductRunnable>
<CommandLineArguments>
<CommandLineArgument
argument = "-s -b"
isEnabled = "YES">
</CommandLineArgument>
<CommandLineArgument
argument = "&quot;Scenario: core::file::Path&quot;"
isEnabled = "NO">
</CommandLineArgument>
<CommandLineArgument
argument = "&quot;Scenario: tjp::core::http::beast&quot;"
isEnabled = "NO">
</CommandLineArgument>
<CommandLineArgument
argument = "&quot;Scenario: tjp::core::http::hirose&quot;"
isEnabled = "NO">
</CommandLineArgument>
<CommandLineArgument
argument = "&quot;Scenario: core::resource_version::GlobalResourceVersion&quot;"
isEnabled = "NO">
</CommandLineArgument>
<CommandLineArgument
argument = "&quot;Scenario: AutoThreadPool&quot;"
isEnabled = "NO">
</CommandLineArgument>
<CommandLineArgument
argument = "&quot;Scenario: base64&quot;"
isEnabled = "NO">
</CommandLineArgument>
<CommandLineArgument
argument = "&quot;Scenario: base32&quot;"
isEnabled = "NO">
</CommandLineArgument>
<CommandLineArgument
argument = "&quot;Scenario: zlib&quot;"
isEnabled = "NO">
</CommandLineArgument>
<CommandLineArgument
argument = "&quot;Scenario: zstd&quot;"
isEnabled = "NO">
</CommandLineArgument>
<CommandLineArgument
argument = "&quot;Scenario: zip::compression&quot;"
isEnabled = "NO">
</CommandLineArgument>
<CommandLineArgument
argument = "&quot;Scenario: const_hash&quot;"
isEnabled = "NO">
</CommandLineArgument>
<CommandLineArgument
argument = "&quot;Scenario: core::internet&quot;"
isEnabled = "NO">
</CommandLineArgument>
<CommandLineArgument
argument = "&quot;Scenario: core::scheduler&quot;"
isEnabled = "NO">
</CommandLineArgument>
<CommandLineArgument
argument = "&quot;Scenario: delegate&quot;"
isEnabled = "NO">
</CommandLineArgument>
<CommandLineArgument
argument = "&quot;Scenario: extensible&quot;"
isEnabled = "NO">
</CommandLineArgument>
</CommandLineArguments>
</LaunchAction>
<ProfileAction
buildConfiguration = "Debug"
shouldUseLaunchSchemeArgsEnv = "YES"
savedToolIdentifier = ""
useCustomWorkingDirectory = "NO"
debugDocumentVersioning = "YES">
<BuildableProductRunnable
runnableDebuggingMode = "0">
<BuildableReference
BuildableIdentifier = "primary"
BlueprintIdentifier = "F608AA9728270442005C276B"
BuildableName = "Core_Misc_Tests"
BlueprintName = "Core_Misc_Tests"
ReferencedContainer = "container:Core_Misc.xcodeproj">
</BuildableReference>
</BuildableProductRunnable>
</ProfileAction>
<AnalyzeAction
buildConfiguration = "Debug">
</AnalyzeAction>
<ArchiveAction
buildConfiguration = "Release"
revealArchiveInOrganizer = "YES">
</ArchiveAction>
</Scheme>

2
Makefile Normal file
View File

@@ -0,0 +1,2 @@
ROOTDIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))..)
include $(ROOTDIR)/Core_Make/tjp/Make/Makefile

7
Makefile.def Executable file
View File

@@ -0,0 +1,7 @@
timprepscius.core_misc.include := -I $(dir $(realpath $(lastword $(MAKEFILE_LIST))))
timprepscius.core_misc.link := -L $(dir $(realpath $(lastword $(MAKEFILE_LIST))))/.bin/$(OBJDIR)
timprepscius.core.include := $(timprepscius.core.include) $(timprepscius.core_misc.include)
timprepscius.core.link := $(timprepscius.core.link) $(timprepscius.core_misc.link)

71
Makefile.project Executable file
View File

@@ -0,0 +1,71 @@
include $(MAKEDIR)/Makefile.base
# use: ls -d tjp/core/*/ tjp/core/*/*/ | rev | cut -c 2- | rev | sed 's/$/ \\/'
PROJECTS := \
tjp/core/args \
tjp/core/bases \
tjp/core/csv \
tjp/core/compression \
tjp/core/delegate \
tjp/core/endian \
tjp/core/events \
tjp/core/extensible \
tjp/core/file \
tjp/core/functions \
tjp/core/hash \
tjp/core/http \
tjp/core/http/hirose \
tjp/core/http/beast \
tjp/core/in_place \
tjp/core/initializer \
tjp/core/internet \
tjp/core/pipe \
tjp/core/profile \
tjp/core/random \
tjp/core/resource_version \
tjp/core/scheduler \
tjp/core/string_lookup \
tjp/core/system_usage \
tjp/core/threads \
tjp/core/thread_pool \
tjp/core/time \
tjp/core/url \
tjp/core/zip \
ifeq (iOS,$(SYS_NAME))
PROJECTS += \
tjp/core/internet/ios \
tjp/core/file/apple \
tjp/core/system_usage/apple \
else
ifeq (Darwin,$(SYS_NAME))
PROJECTS += \
tjp/core/internet/curl \
tjp/core/file/apple \
tjp/core/system_usage/apple \
# tjp/core/internet/curl \
else
PROJECTS += \
tjp/core/internet/curl \
tjp/core/file/linux \
tjp/core/system_usage/linux \
tjp/core/bases/linux \
endif
endif
#SRC_PCH := tjp/core/Precompile.pch
INCPATH := \
$(timprepscius.libraries.cpp.include) \
$(timprepscius.core.include)
LIBFILE := libCore_Misc.a
COPYTO := $(LIBRARIES_PROJECT)
include $(MAKEDIR)/Makefile.lib

1
tests/Core_Misc Symbolic link
View File

@@ -0,0 +1 @@
..

2
tests/Makefile Normal file
View File

@@ -0,0 +1,2 @@
ROOTDIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))../..)
include $(ROOTDIR)/Core_Make/tjp/Make/Makefile

45
tests/Makefile.project Executable file
View File

@@ -0,0 +1,45 @@
include $(MAKEDIR)/Makefile.base
# use: ls -d core/*/ core/*/*/ | rev | cut -c 2- | rev | sed 's/$/ \\/'
BASE_PROJECT=Core_Misc
PROJECTS := \
. \
$(BASE_PROJECT)/tjp/core/http/hirose/_test \
$(BASE_PROJECT)/tjp/core/http/beast/_test \
$(BASE_PROJECT)/tjp/core/bases/_tests \
$(BASE_PROJECT)/tjp/core/containers/_tests \
$(BASE_PROJECT)/tjp/core/compression/_tests \
$(BASE_PROJECT)/tjp/core/csv/_tests \
$(BASE_PROJECT)/tjp/core/delegate/_tests \
$(BASE_PROJECT)/tjp/core/extensible/_tests \
$(BASE_PROJECT)/tjp/core/pipe/_tests \
$(BASE_PROJECT)/tjp/core/resource_version/_tests \
$(BASE_PROJECT)/tjp/core/thread_pool/_tests \
$(BASE_PROJECT)/tjp/core/url/_tests \
$(BASE_PROJECT)/tjp/core/zip/_tests \
INCPATH := \
$(timprepscius.libraries.cpp.include) \
$(timprepscius.core.include)
LDPATH := $(timprepscius.libraries.cpp.link)
LIBS := \
-lCore_Future \
-lCore_Misc \
-lCore_Zero \
-lz_ \
-lzstd
EXEFILE := Core_Misc_Tests.exe
#COPYTO := $(ROOTDIR)/.bin
ifeq (Darwin,$(SYS_NAME))
LIBS += -lc++
endif
include $(MAKEDIR)/Makefile.bin

19
tests/main.cpp Normal file
View File

@@ -0,0 +1,19 @@
// TJP COPYRIGHT HEADER
#define CATCH_CONFIG_RUNNER
#include <tjp/core/testing/catch.hpp>
#include <tjp/core/log/Log.h>
using namespace tjp;
using namespace core;
int main( int argc, char* argv[] )
{
xLogInitialize("Core_Misc_Tests.txt");
xLogActivateStory("testing");
xLogActivateStory("debug");
int result = Catch::Session().run( argc, argv );
return result;
}

42
tjp/core/algorithm/Sortable.hpp Normal file
View File

@@ -0,0 +1,42 @@
// TJP COPYRIGHT HEADER
#pragma once
namespace tjp::core {
template<typename T, typename S>
struct Sortable
{
T t;
S s;
bool operator<(const Sortable &rhs) const
{
return s < rhs.s;
}
} ;
template<typename T, typename F>
struct SortableFunc
{
T t;
bool operator<(const SortableFunc &rhs) const
{
return f(t) < f(rhs.t);
}
} ;
template<typename T, typename S>
struct SortablePtr
{
T t;
S *s;
bool operator<(const SortablePtr &rhs) const
{
return *s < *rhs.s;
}
} ;
} // namespace

27
tjp/core/algorithm/_tests/call_with_pairs.cpp Normal file
View File

@@ -0,0 +1,27 @@
// TJP COPYRIGHT HEADER
#include <tjp/core/testing/catch.hpp>
#include "../call_with_pairs.hpp"
namespace tjp::core {
namespace {
SCENARIO("core::call_with_pairs")
{
WHEN("a function is called with pairs")
{
int c = 0;
auto f = [&](auto a, auto b)
{
return c += a * b;
};
call_with_pairs(f, 1, 2, 3, 4, 5, 6);
REQUIRE(c == 44);
}
}
}
} // namespace

43
tjp/core/algorithm/call_with_pairs.hpp Normal file
View File

@@ -0,0 +1,43 @@
// TJP COPYRIGHT HEADER
#pragma once
#include <utility>
#include <tuple>
#include <type_traits>
#include <functional> // for std::invoke
namespace tjp::core {
template<typename F, typename ObjOrFirstArg, typename... Args>
constexpr void call_with_pairs(F&& f, ObjOrFirstArg&& obj_or_first, Args&&... args) {
constexpr bool is_member = std::is_member_function_pointer_v<std::decay_t<F>>;
if constexpr (is_member) {
static_assert(sizeof...(Args) % 2 == 0,
"Member function call_with_pairs: arguments after object must be in pairs");
} else {
static_assert((sizeof...(Args)+1) % 2 == 0,
"Non-member call_with_pairs: all arguments must be in pairs");
}
if constexpr (is_member) {
auto tup = std::forward_as_tuple(std::forward<Args>(args)...);
[&]<std::size_t... I>(std::index_sequence<I...>) {
(std::invoke(f,
std::forward<ObjOrFirstArg>(obj_or_first),
std::get<2*I>(tup),
std::get<2*I+1>(tup)), ...);
}(std::make_index_sequence<sizeof...(Args) / 2>{});
} else {
auto tup = std::forward_as_tuple(std::forward<ObjOrFirstArg>(obj_or_first),
std::forward<Args>(args)...);
[&]<std::size_t... I>(std::index_sequence<I...>) {
(std::invoke(f,
std::get<2*I>(tup),
std::get<2*I+1>(tup)), ...);
}(std::make_index_sequence<(sizeof...(Args) + 1) / 2>{});
}
}
} // namespace

57
tjp/core/algorithm/find_if_random.hpp Normal file
View File

@@ -0,0 +1,57 @@
// TJP COPYRIGHT HEADER
#pragma once
#include <algorithm>
#include <vector>
#include <tjp/core/type_traits/is_comparable.hpp>
#include <optional>
#include <tjp/core/random/Random.h>
namespace tjp {
namespace core {
template<typename T, typename F, typename F1>
auto find_get_if_random_transform (T &c, F &&f, F1 &&transform)
{
std::vector<typename T::const_iterator> items;
for (auto i=c.begin(); i!=c.end(); ++i)
{
if (f(*i))
items.push_back(i);
}
using value_type = decltype(transform(*items.front()));
if (items.empty())
return std::optional<value_type>();
auto index = StandardRandom.nextInt<size_t>(0,items.size()-1);
return std::optional<value_type>(transform(*items[index]));
}
template<typename T, typename F>
auto find_get_if_random (T &c, F &&f)
{
std::vector<typename T::const_iterator> items;
for (auto i=c.begin(); i!=c.end(); ++i)
{
if (f(*i))
items.push_back(i);
}
using value_type = typename T::value_type;
if (items.empty())
return std::optional<value_type>();
auto index = StandardRandom.nextInt<size_t>(0,items.size()-1);
return std::optional<value_type>(*items[index]);
}
} // namespace utilities
} // namespace

17
tjp/core/algorithm/for_each_argument.hpp Normal file
View File

@@ -0,0 +1,17 @@
// TJP COPYRIGHT HEADER
// https://stackoverflow.com/questions/12030538/calling-a-function-for-each-variadic-template-argument-and-an-array
#pragma once
namespace tjp {
namespace core {
template <class F, class... Args>
void for_each_argument(F f, Args&&... args) {
[](...){}((f(std::forward<Args>(args)), 0)...);
}
} // namespace core
} // namespace

16
tjp/core/algorithm/reverse_inplace.hpp Normal file
View File

@@ -0,0 +1,16 @@
// TJP COPYRIGHT HEADER
#pragma once
namespace tjp::core {
template<typename T>
T reverse_return(T &&c)
{
for (auto i=0; i<c.size()/2; ++i)
std::swap(c[i], c[c.size()-(i+1)]);
return c;
}
} // namespace

14
tjp/core/algorithm/xor_inplace.hpp Normal file
View File

@@ -0,0 +1,14 @@
// TJP COPYRIGHT HEADER
#pragma once
namespace tjp::core {
template<typename T, typename U>
void xor_inplace(T &t, U u)
{
for (auto &v : t)
v ^= u;
}
} // namespace

42
tjp/core/args/Args.cpp Normal file
View File

@@ -0,0 +1,42 @@
// TJP COPYRIGHT HEADER
#ifdef TJP_CORE_HEADER_ONLY
#pragma once
#endif
#include <tjp/core/header_only/compile.h>
#include "Args.h"
#include <tjp/core/string/starts_with.hpp>
namespace tjp::core {
TJP_CORE_HEADER_ONLY_INLINE
std::map<std::string, std::string> mapArgs(int argc, const char *argv[])
{
std::map<std::string, std::string> mapped;
static const std::string prefix = "--";
for (int i=1; i<argc; ++i)
{
std::string s = argv[i];
if (starts_with(s, prefix))
{
bool hasValue = (i+1)<argc && !starts_with(std::string(argv[i+1]), prefix);
std::string key = s.substr(2);
if (hasValue)
{
std::string value = argv[i+1];
mapped[key] = value;
i++;
}
else
{
mapped[key] = "";
}
}
}
return mapped;
}
} // namespace

67
tjp/core/args/Args.h Normal file
View File

@@ -0,0 +1,67 @@
// TJP COPYRIGHT HEADER
#pragma once
#include <tjp/core/string/from_string.hpp>
#include <tjp/core/string/cmp_case.hpp>
#include <tjp/core/containers/Optional.hpp>
#include <map>
namespace tjp {
namespace core {
typedef std::map<std::string, std::string> Args;
std::map<std::string, std::string> mapArgs(int argc, const char *argv[]);
template<typename T>
inline T argOrDefault(const std::map<std::string, std::string> &args, const std::string &key, const T &default_)
{
auto i = args.find(key);
if (i == args.end())
{
return default_;
}
return core::from_string<T>(i->second);
}
template<>
inline bool argOrDefault(const std::map<std::string, std::string> &args, const std::string &key, const bool &default_)
{
auto i = args.find(key);
if (i == args.end())
{
return default_;
}
return equal_case(i->second, "true");
}
inline std::string argOrDefault_(const std::map<std::string, std::string> &args, const std::string &key, const char *default_)
{
auto i = args.find(key);
if (i == args.end())
{
return default_;
}
return i->second;
}
inline Optional<std::string> argOptional(const std::map<std::string, std::string> &args, const std::string &key)
{
auto i = args.find(key);
if (i != args.end())
return i->second;
return {};
}
} // namespace
} // namespace
#ifdef TJP_CORE_HEADER_ONLY
#include "Args.cpp"
#endif

60
tjp/core/args/ArgsFile.cpp Normal file
View File

@@ -0,0 +1,60 @@
// TJP COPYRIGHT HEADER
#ifdef TJP_CORE_HEADER_ONLY
#pragma once
#endif
#include <tjp/core/header_only/compile.h>
#include "ArgsFile.h"
#include <tjp/core/exception/Exception.hpp>
#include <tjp/core/string/String.hpp>
#include <tjp/core/string/StringView.hpp>
#include <tjp/core/string/trim.h>
#include <fstream>
#include <string>
#include <map>
namespace tjp::core {
TJP_CORE_HEADER_ONLY_INLINE
Optional<Args> readArgsFile(const std::string& filename)
{
std::ifstream file(filename);
if (!file)
throw Exception("Failed to open file");
Args result;
String line_;
while (std::getline(file, line_))
{
auto line = trim_copy(StringView(line_));
// Skip empty lines and comments
if (line.empty() || line[0] == '#')
continue;
auto pos = line.find('=');
if (pos == line.npos)
continue;
auto key = line.substr(0, pos);
auto value = line.substr(pos + 1);
result[String(key)] = String(value);
}
return result;
}
TJP_CORE_HEADER_ONLY_INLINE
void mergeArgs(Args &args, Args &rhs)
{
args.merge(rhs);
}
} // namespace

16
tjp/core/args/ArgsFile.h Normal file
View File

@@ -0,0 +1,16 @@
// TJP COPYRIGHT HEADER
#pragma once
#include "Args.h"
namespace tjp::core {
Optional<Args> readArgsFile(const String &fileName);
void mergeArgs(Args &args, Args &rhs);
} // namespace
#ifdef TJP_CORE_HEADER_ONLY
#include "ArgsFile.cpp"
#endif

96
tjp/core/bases/Base16.cpp Executable file
View File

@@ -0,0 +1,96 @@
// TJP COPYRIGHT HEADER
#ifdef TJP_CORE_HEADER_ONLY
#pragma once
#endif
#include <tjp/core/header_only/compile.h>
#include "Base16.h"
#include <resolv.h>
#include <memory.h>
#include <assert.h>
#include <tjp/core/assert/debug_assert.h>
#include <tjp/core/assert/handle_assert.hpp>
namespace tjp::core {
using b16d = b16_bin;
TJP_CORE_HEADER_ONLY_INLINE
size_t toBase16 (const b16d *in, size_t inSize, b16_char *out, size_t outSize)
{
const char *base16_charset = "0123456789abcdef";
handle_assert(outSize == 2 * inSize) {
return 0;
}
auto begin = in;
auto end = in + inSize;
for (auto *i = begin; i!=end; ++i)
{
int n1 = (int)*i & 0x0F;
int n2 = (int)*i >> 4 & 0x0F;
*out++ = base16_charset[n2];
*out++ = base16_charset[n1];
}
return 2*inSize;
}
TJP_CORE_HEADER_ONLY_INLINE
std::string toBase16 (const b16d *dataToEncode, size_t dataToEncodeLength)
{
std::string encodedData;
encodedData.resize(dataToEncodeLength*2);
toBase16(dataToEncode, dataToEncodeLength, encodedData.data(), encodedData.size());
return encodedData;
}
inline int getNibble (char c)
{
if (c >= '0' && c <= '9')
return c - '0';
if (c >= 'a' && c <= 'f')
return c - 'a' + 10;
if (c >= 'A' && c <= 'F')
return c - 'A' + 10;
handle_assert(false)
;
return 0;
}
TJP_CORE_HEADER_ONLY_INLINE
size_t fromBase16 (const char *in, size_t inSize, b16d *out, size_t outSize)
{
handle_assert((inSize % 2 == 0) && (inSize == 2 * outSize)) {
return 0;
}
auto begin = in;
auto end = in + inSize;
for (auto *i = begin; i!=end; )
{
int n2 = getNibble(*i++);
int n1 = getNibble(*i++);
unsigned char c = n2 << 4 | n1;
*out++ = c;
}
return outSize;
}
} // namespace

104
tjp/core/bases/Base16.h Executable file
View File

@@ -0,0 +1,104 @@
// TJP COPYRIGHT HEADER
#pragma once
#include <string>
#include <vector>
#include <tjp/core/types/Types.h>
namespace tjp::core {
using b16_bin = u8;
using b16_char = char;
size_t fromBase16 (const b16_char *in, size_t inSize, b16_bin *out, size_t outSize);
inline
size_t base16EncodeSize(size_t size)
{
return size * 2;
}
inline
size_t base16DecodeSize(const b16_char *b, size_t size)
{
return size / 2;
}
template<typename T=b16_char>
std::vector<T> fromBase16 (const b16_char *b, size_t size)
{
auto decodeSize = base16DecodeSize(b, size);
std::vector<T> r(decodeSize / sizeof(T));
fromBase16((const b16_char *)b, size, (b16_bin *)r.data(), decodeSize);
return r;
}
template<typename U>
U fromBase16v (const b16_char *b, size_t size)
{
auto decodeSize = base16DecodeSize(b, size);
U r(decodeSize / sizeof(typename U::value_type));
fromBase16((const b16_char *)b, size, (b16_bin *)r.data(), decodeSize);
return r;
}
template<typename T=b16_char>
std::vector<T> fromBase16 (const std::string_view &block)
{
return fromBase16<T>(block.data(), block.size());
}
template<typename T>
bool fromBase16(T &t, const b16_char *b, size_t size)
{
return fromBase16(b, size, (b16_bin *)&t, sizeof(t)) == sizeof(t);
}
template<typename T>
bool fromBase16(T &t, const std::string_view &block)
{
return fromBase16(t, block.data(), block.size());
}
std::string toBase16 (const b16_bin *b, size_t size);
size_t toBase16 (const b16_bin *in, size_t inSize, b16_char *out, size_t outSize);
template<typename T>
std::string toBase16 (const T *b, size_t size)
{
return toBase16((const b16_bin *)b, sizeof(*b) * size);
}
//template<typename T>
//std::string toBase16 (const std::vector<T> &block)
//{
// return toBase16((b16_bin *)block.data(), sizeof(block[0]) * block.size());
//}
template<typename T>
std::string toBase16 (const T &block)
{
return toBase16((b16_bin *)block.data(), sizeof(typename T::value_type) * block.size());
}
template<typename T>
bool toBase16(const T &t, b16_char *out, size_t outSize)
{
return toBase16((b16_bin *)&t, sizeof(t), out, outSize) == outSize;
}
template<typename T>
bool toBase16(const T &t, const std::string_view &block)
{
return toBase16(t, (b16_char *)block.data(), block.size());
}
} // namespace
#ifdef TJP_CORE_HEADER_ONLY
#include "Base16.cpp"
#endif

88
tjp/core/bases/Base32.cpp Executable file
View File

@@ -0,0 +1,88 @@
// TJP COPYRIGHT HEADER
#include "Base32.h"
#include <base32/base32_crockford.hpp>
#include <base32/base32_rfc4648.hpp>
#include <resolv.h>
#include <memory.h>
#include <assert.h>
#include <tjp/core/assert/debug_assert.h>
#include <tjp/core/assert/handle_assert.hpp>
namespace tjp::core {
//using base32 = cppcodec::base32_rfc4648;
//using base32base = cppcodec::detail::base32_rfc4648;
using base32 = cppcodec::base32_crockford;
using base32base = cppcodec::detail::base32_crockford;
size_t base32EncodeSize(size_t size)
{
return base32::encoded_size(size);
}
size_t base32DecodeSize(const b32_char *b, size_t size)
{
auto maxSize = base32::decoded_max_size(size);
auto actualSize = maxSize;
if constexpr (base32base::generates_padding())
{
auto end = size;
auto i = end;
for (; i>0; --i)
{
if (!base32base::is_padding_symbol(b[i-1]))
break;
}
auto numPadding = end - i;
int paddingRemove = 0;
switch (numPadding) {
case 1:
case 2: paddingRemove = 1; break;
case 3: paddingRemove = 2; break;
case 4: paddingRemove = 3; break;
case 5:
case 6: paddingRemove = 4; break;
case 7:
case 8: paddingRemove = 5; break;
}
actualSize = maxSize - paddingRemove;
}
return actualSize;
}
using b32d = b32_bin;
size_t toBase32 (const b32d *in, size_t inSize, b32_char *out, size_t outSize)
{
auto size = base32::encode(out, outSize, in, inSize);
return size;
}
std::string toBase32 (const b32d *dataToEncode, size_t dataToEncodeLength)
{
std::string encodedData;
encodedData.resize(dataToEncodeLength*2);
return base32::encode(dataToEncode, dataToEncodeLength);
}
size_t fromBase32 (const char *in, size_t inSize, b32d *out, size_t outSize)
{
debug_assert(outSize == base32DecodeSize(in, inSize));
auto size = base32::decode(out, base32::decoded_max_size(inSize), in, inSize);
return size;
}
} // namespace

91
tjp/core/bases/Base32.h Executable file
View File

@@ -0,0 +1,91 @@
// TJP COPYRIGHT HEADER
#pragma once
#include <string>
#include <vector>
#include <tjp/core/types/Types.h>
namespace tjp::core {
using b32_bin = u8;
using b32_char = char;
size_t fromBase32 (const b32_char *in, size_t inSize, b32_bin *out, size_t outSize);
size_t base32EncodeSize(size_t size);
size_t base32DecodeSize(const b32_char *b, size_t size);
template<typename T=b32_char>
std::vector<T> fromBase32 (const b32_char *b, size_t size)
{
auto decodeSize = base32DecodeSize(b, size);
std::vector<T> r(decodeSize / sizeof(T));
fromBase32((const b32_char *)b, size, (b32_bin *)r.data(), decodeSize);
return r;
}
template<typename U>
U fromBase32v (const b32_char *b, size_t size)
{
auto decodeSize = base32DecodeSize(b, size);
U r(decodeSize / sizeof(typename U::value_type));
fromBase32((const b32_char *)b, size, (b32_bin *)r.data(), decodeSize);
return r;
}
template<typename T=b32_char>
std::vector<T> fromBase32 (const std::string_view &block)
{
return fromBase32<T>(block.data(), block.size());
}
template<typename T>
bool fromBase32(T &t, const b32_char *b, size_t size)
{
return fromBase32(b, size, (b32_bin *)&t, sizeof(t)) == sizeof(t);
}
template<typename T>
bool fromBase32(T &t, const std::string_view &block)
{
return fromBase32(t, block.data(), block.size());
}
std::string toBase32 (const b32_bin *b, size_t size);
size_t toBase32 (const b32_bin *in, size_t inSize, b32_char *out, size_t outSize);
template<typename T>
std::string toBase32 (const T *b, size_t size)
{
return toBase32((const b32_bin *)b, sizeof(*b) * size);
}
//template<typename T>
//std::string toBase32 (const std::vector<T> &block)
//{
// return toBase32((b32_bin *)block.data(), sizeof(block[0]) * block.size());
//}
template<typename T>
std::string toBase32 (const T &block)
{
return toBase32((b32_bin *)block.data(), sizeof(typename T::value_type) * block.size());
}
template<typename T>
bool toBase32(const T &t, b32_char *out, size_t outSize)
{
return toBase32((b32_bin *)&t, sizeof(t), out, outSize) == outSize;
}
template<typename T>
bool toBase32(const T &t, const std::string_view &block)
{
return toBase32(t, (b32_char *)block.data(), block.size());
}
} // namespace

362
tjp/core/bases/Base64.cpp Executable file
View File

@@ -0,0 +1,362 @@
// TJP COPYRIGHT HEADER
#ifdef TJP_CORE_HEADER_ONLY
#pragma once
#endif
#include <tjp/core/header_only/compile.h>
#include "Base64.h"
//#include <zlib/zlib.h>
//#include <resolv.h>
//#include <memory.h>
#include <string.h>
#include <tjp/core/assert/debug_assert.h>
#include <tjp/core/iterators/range.hpp>
namespace tjp {
namespace core {
namespace detail {
static const char Base64[] =
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
static const char Pad64 = '=';
#define Assert debug_assert
using u_char = unsigned char;
using u_int = unsigned int;
TJP_CORE_HEADER_ONLY_INLINE
int b64_ntop(u_char const *src, size_t srclength, char *target, size_t targsize) {
size_t datalength = 0;
u_char input[3];
u_char output[4];
size_t i;
while (2 < srclength) {
input[0] = *src++;
input[1] = *src++;
input[2] = *src++;
srclength -= 3;
output[0] = input[0] >> 2;
output[1] = ((input[0] & 0x03) << 4) + (input[1] >> 4);
output[2] = ((input[1] & 0x0f) << 2) + (input[2] >> 6);
output[3] = input[2] & 0x3f;
Assert(output[0] < 64);
Assert(output[1] < 64);
Assert(output[2] < 64);
Assert(output[3] < 64);
if (datalength + 4 > targsize)
return (-1);
target[datalength++] = Base64[output[0]];
target[datalength++] = Base64[output[1]];
target[datalength++] = Base64[output[2]];
target[datalength++] = Base64[output[3]];
}
/* Now we worry about padding. */
if (0 != srclength) {
/* Get what's left. */
input[0] = input[1] = input[2] = '\0';
for (i = 0; i < srclength; i++)
input[i] = *src++;
output[0] = input[0] >> 2;
output[1] = ((input[0] & 0x03) << 4) + (input[1] >> 4);
output[2] = ((input[1] & 0x0f) << 2) + (input[2] >> 6);
Assert(output[0] < 64);
Assert(output[1] < 64);
Assert(output[2] < 64);
if (datalength >= targsize)
return -1;
target[datalength++] = Base64[output[0]];
if (datalength >= targsize)
return -1;
target[datalength++] = Base64[output[1]];
if (srclength != 1)
{
if (datalength >= targsize)
return -1;
target[datalength++] = Base64[output[2]];
}
}
if (datalength > targsize)
return (-1);
if (targsize > datalength)
target[datalength] = '\0'; /* Returned value doesn't count \0. */
return (int)datalength;
}
/*
TJP, man I can't believe I had to do this.
So I made the b64_pton_l work with sizes for input, and outputs which don't have \0
*/
TJP_CORE_HEADER_ONLY_INLINE
int b64_pton_l(char const *src, size_t srcLength, u_char *target, size_t targsize)
{
const char *end = src + srcLength;
u_int tarindex, state;
int ch;
const char *pos;
state = 0;
tarindex = 0;
while (src != end && (ch = *src++) != '\0') {
if (isspace(ch)) /* Skip whitespace anywhere. */
continue;
if (ch == Pad64)
break;
pos = strchr(Base64, ch);
if (pos == 0) /* A non-base64 character. */
return (-1);
switch (state) {
case 0:
if (target) {
if (tarindex >= targsize)
return (-1);
target[tarindex] = (pos - Base64) << 2;
}
state = 1;
break;
case 1:
if (target) {
target[tarindex] |= (pos - Base64) >> 4;
if (tarindex + 1 >= targsize)
return (-1);
target[tarindex+1] = ((pos - Base64) & 0x0f)
<< 4 ;
}
tarindex++;
state = 2;
break;
case 2:
if (target) {
target[tarindex] |= (pos - Base64) >> 2;
if (tarindex + 1 >= targsize)
return (-1);
target[tarindex+1] = ((pos - Base64) & 0x03)
<< 6;
}
tarindex++;
state = 3;
break;
case 3:
if (target) {
if (tarindex >= targsize)
return (-1);
target[tarindex] |= (pos - Base64);
}
tarindex++;
state = 0;
break;
}
}
/*
* We are done decoding Base-64 chars. Let's see if we ended
* on a byte boundary, and/or with erroneous trailing characters.
*/
// if (ch == Pad64) { /* We got a pad char. */
// ch = *src++; /* Skip it, get next. */
// switch (state) {
// case 0: /* Invalid = in first position */
// case 1: /* Invalid = in second position */
// return (-1);
//
// case 2: /* Valid, means one byte of info */
// /* Skip any number of spaces. */
// for (; ch != '\0'; ch = *src++)
// if (!isspace(ch))
// break;
// /* Make sure there is another trailing = sign. */
// if (ch != Pad64)
// return (-1);
// ch = *src++; /* Skip the = */
// /* Fall through to "single trailing =" case. */
// /* FALLTHROUGH */
//
// case 3: /* Valid, means two bytes of info */
// /*
// * We know this char is an =. Is there anything but
// * whitespace after it?
// */
// for (; ch != '\0'; ch = *src++)
// if (!isspace(ch))
// return (-1);
//
// /*
// * Now make sure for cases 2 and 3 that the "extra"
// * bits that slopped past the last full byte were
// * zeros. If we don't check them, they become a
// * subliminal channel.
// */
// if (target && target[tarindex] != 0)
// return (-1);
// }
// } else {
// /*
// * We ended by seeing the end of the string. Make sure we
// * have no partial bytes lying around.
// */
// if (state != 0)
// return (-1);
// }
return (tarindex);
}
} // namespace
TJP_CORE_HEADER_ONLY_INLINE
size_t base64EncodeSize(size_t v)
{
if (v == 0)
return 0;
auto vm1 = v-1;
// 1 = 2 = 0 * 4 + 2
// 2 = 3 = 0 * 4 + 2 + 1
// 3 = 4 = 0 * 4 + 2 + 2
// 4 = 6 = 1 * 4 + 2 + 0
// 5 = 7 = 1 * 4 + 2 + 1
// 6 = 8
// 7 = 10 = 6+4
return (vm1 / 3 * 4 + 2) + (vm1 % 3);
}
TJP_CORE_HEADER_ONLY_INLINE
size_t base64DecodeSize(size_t v)
{
return v * 3 / 4;
}
TJP_CORE_HEADER_ONLY_INLINE
std::string toBase64 (const std::vector<char> &dataToEncode)
{
return toBase64(dataToEncode.data(), (int)dataToEncode.size());
}
TJP_CORE_HEADER_ONLY_INLINE
std::string toBase64 (const std::string_view &dataToEncode)
{
return toBase64(dataToEncode.data(), (int)dataToEncode.size());
}
TJP_CORE_HEADER_ONLY_INLINE
std::string toBase64 (const std::string &dataToEncode)
{
return toBase64(dataToEncode.data(), (int)dataToEncode.size());
}
TJP_CORE_HEADER_ONLY_INLINE
size_t toBase64 (const char *in, size_t inSize, char *out, size_t outSize)
{
return detail::b64_ntop(
(const unsigned char *)in, inSize,
out, outSize
);
}
TJP_CORE_HEADER_ONLY_INLINE
std::string toBase64 (const char *dataToEncode, size_t dataToEncodeLength)
{
std::string encodedData;
auto encodeSize = base64EncodeSize(dataToEncodeLength);
encodedData.resize(encodeSize);
auto encodedRealLength = toBase64(
dataToEncode, dataToEncodeLength,
encodedData.data(), encodeSize
);
debug_assert(encodedRealLength == encodeSize);
if (encodedRealLength == -1)
return {};
return encodedData;
}
TJP_CORE_HEADER_ONLY_INLINE
void padBase64(std::string &v)
{
auto paddingLength = (4 - (v.size() % 4)) % 4;
for (auto _ : range(paddingLength))
v.push_back('=');
}
TJP_CORE_HEADER_ONLY_INLINE
std::vector<char> fromBase64 (const std::string_view &dataToDecode)
{
return fromBase64(dataToDecode.data(), (int)dataToDecode.size());
}
TJP_CORE_HEADER_ONLY_INLINE
size_t fromBase64(const char *in, size_t inSize, char *out, size_t outSize)
{
return detail::b64_pton_l(
in, inSize,
(unsigned char *)out, outSize
);
}
TJP_CORE_HEADER_ONLY_INLINE
size_t fromBase64(const std::string_view &in, char *out, size_t outSize)
{
return detail::b64_pton_l(
in.data(), in.size(),
(unsigned char *)out, outSize
);
}
TJP_CORE_HEADER_ONLY_INLINE
std::vector<char> fromBase64 (const char *dataToDecode, size_t dataToDecodeLength)
{
std::vector<char> decodedData;
auto decodedBufferLength = dataToDecodeLength * 3 / 4 + 1 + 128;
decodedData.resize(decodedBufferLength);
auto decodedBufferRealLength = fromBase64(
dataToDecode, dataToDecodeLength,
decodedData.data(),
decodedBufferLength
);
debug_assert(decodedBufferRealLength <= base64DecodeSize(dataToDecodeLength));
decodedData.resize(decodedBufferRealLength);
return decodedData;
}
} // namespace
} // namespace

46
tjp/core/bases/Base64.h Executable file
View File

@@ -0,0 +1,46 @@
// TJP COPYRIGHT HEADER
#pragma once
#include <string>
#include <string_view>
#include <vector>
namespace tjp::core {
void padBase64 (std::string &);
size_t base64EncodeSize(size_t);
size_t base64DecodeSize(size_t);
std::string toBase64 (const std::vector<char> &block);
std::string toBase64 (const std::string_view &block);
std::string toBase64 (const std::string &block);
std::vector<char> fromBase64 (const std::string_view &b64);
std::string toBase64 (const char *b, size_t size);
std::vector<char> fromBase64 (const char *b, size_t size);
size_t toBase64 (const char *in, size_t inSize, char *out, size_t outSize);
size_t fromBase64(const char *in, size_t inSize, char *out, size_t outSize);
size_t fromBase64(const std::string_view &in, char *out, size_t outSize);
template<typename T>
T fromBase64(const std::string_view &in)
{
T t;
fromBase64((const char *)in.data(), in.size(), (char *)&t, sizeof(t));
return t;
}
template<typename T>
std::string toBase64(const T &t)
{
return toBase64((const char *)&t, sizeof(t));
}
} // namespace
#ifdef TJP_CORE_HEADER_ONLY
#include "Base64.cpp"
#endif

114
tjp/core/bases/Base64URL.cpp Executable file
View File

@@ -0,0 +1,114 @@
// TJP COPYRIGHT HEADER
#ifdef TJP_CORE_HEADER_ONLY
#pragma once
#endif
#include <tjp/core/header_only/compile.h>
#include "Base64URL.h"
namespace tjp::core {
/*
Base64 translates 24 bits into 4 ASCII characters at a time. First,
3 8-bit bytes are treated as 4 6-bit groups. Those 4 groups are
translated into ASCII characters. That is, each 6-bit number is treated
as an index into the ASCII character array.
If the final set of bits is less 8 or 16 instead of 24, traditional base64
would add a padding character. However, if the length of the data is
known, then padding can be eliminated.
One difference between the "standard" Base64 is two characters are different.
See RFC 4648 for details.
This is how we end up with the Base64 URL encoding.
*/
const char base64_url_alphabet[] = {
'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M',
'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_'
};
TJP_CORE_HEADER_ONLY_INLINE
std::string toBase64URL(const char *in, int len) {
std::string out;
int val =0, valb=-6;
unsigned int i = 0;
for (i = 0; i < len; i++) {
unsigned char c = in[i];
val = (val<<8) + c;
valb += 8;
while (valb >= 0) {
out.push_back(base64_url_alphabet[(val>>valb)&0x3F]);
valb -= 6;
}
}
if (valb > -6) {
out.push_back(base64_url_alphabet[((val<<8)>>(valb+8))&0x3F]);
}
return out;
}
TJP_CORE_HEADER_ONLY_INLINE
std::string toBase64URL(const char *in, size_t len) {
return toBase64URL(in, (int)len);
}
TJP_CORE_HEADER_ONLY_INLINE
std::vector<char> fromBase64URL(const char *in, int len) {
std::vector<char> out;
static std::vector<int> T;
if (T.empty())
{
T.resize(256, -1);
for (auto i =0; i < 64; i++) T[base64_url_alphabet[i]] = i;
}
unsigned int i;
int val = 0, valb = -8;
for (i = 0; i < len; i++) {
unsigned char c = in[i];
if (T[c] == -1) break;
val = (val<<6) + T[c];
valb += 6;
if (valb >= 0) {
out.push_back(char((val>>valb)&0xFF));
valb -= 8;
}
}
return out;
}
TJP_CORE_HEADER_ONLY_INLINE
std::vector<char> fromBase64URL(const char *in, size_t len) {
return fromBase64URL(in, (int)len);
}
TJP_CORE_HEADER_ONLY_INLINE
std::string toBase64URL (const std::vector<char> &block)
{
return toBase64URL(block.data(), block.size());
}
TJP_CORE_HEADER_ONLY_INLINE
std::string toBase64URL (const std::string_view &block)
{
return toBase64URL(block.data(), block.size());
}
TJP_CORE_HEADER_ONLY_INLINE
std::vector<char> fromBase64URL (const std::string_view &b64)
{
return fromBase64URL(b64.data(), b64.size());
}
} // namespace

22
tjp/core/bases/Base64URL.h Executable file
View File

@@ -0,0 +1,22 @@
// TJP COPYRIGHT HEADER
#pragma once
#include <string>
#include <string_view>
#include <vector>
namespace tjp::core {
std::string toBase64URL (const std::vector<char> &block);
std::string toBase64URL (const std::string_view &block);
std::vector<char> fromBase64URL (const std::string_view &b64);
std::string toBase64URL (const char *b, size_t size);
std::vector<char> fromBase64URL (const char *b, size_t size);
} // namespace
#ifdef TJP_CORE_HEADER_ONLY
#include "Base64URL.cpp"
#endif

310
tjp/core/bases/_remove/ntop.c Executable file
View File

@@ -0,0 +1,310 @@
/*
* Copyright (c) 1996-1999 by Internet Software Consortium.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SOFTWARE CONSORTIUM DISCLAIMS
* ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL INTERNET SOFTWARE
* CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
* DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
* PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
* ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS
* SOFTWARE.
*/
/*
* Portions Copyright (c) 1995 by International Business Machines, Inc.
*
* International Business Machines, Inc. (hereinafter called IBM) grants
* permission under its copyrights to use, copy, modify, and distribute this
* Software with or without fee, provided that the above copyright notice and
* all paragraphs of this notice appear in all copies, and that the name of IBM
* not be used in connection with the marketing of any product incorporating
* the Software or modifications thereof, without specific, written prior
* permission.
*
* To the extent it has a right to do so, IBM grants an immunity from suit
* under its patents, if any, for the use, sale or manufacture of products to
* the extent that such products are used for performing Domain Name System
* dynamic updates in TCP/IP networks by means of the Software. No immunity is
* granted for any product per se or for any other function of any product.
*
* THE SOFTWARE IS PROVIDED "AS IS", AND IBM DISCLAIMS ALL WARRANTIES,
* INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
* PARTICULAR PURPOSE. IN NO EVENT SHALL IBM BE LIABLE FOR ANY SPECIAL,
* DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER ARISING
* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE, EVEN
* IF IBM IS APPRISED OF THE POSSIBILITY OF SUCH DAMAGES.
*/
#include <sys/types.h>
#include <sys/param.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <arpa/nameser.h>
#include <ctype.h>
#include <resolv.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#define Assert(Cond) if (!(Cond)) abort()
static const char Base64[] =
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
static const char Pad64 = '=';
/* (From RFC1521 and draft-ietf-dnssec-secext-03.txt)
The following encoding technique is taken from RFC 1521 by Borenstein
and Freed. It is reproduced here in a slightly edited form for
convenience.
A 65-character subset of US-ASCII is used, enabling 6 bits to be
represented per printable character. (The extra 65th character, "=",
is used to signify a special processing function.)
The encoding process represents 24-bit groups of input bits as output
strings of 4 encoded characters. Proceeding from left to right, a
24-bit input group is formed by concatenating 3 8-bit input groups.
These 24 bits are then treated as 4 concatenated 6-bit groups, each
of which is translated into a single digit in the base64 alphabet.
Each 6-bit group is used as an index into an array of 64 printable
characters. The character referenced by the index is placed in the
output string.
Table 1: The Base64 Alphabet
Value Encoding Value Encoding Value Encoding Value Encoding
0 A 17 R 34 i 51 z
1 B 18 S 35 j 52 0
2 C 19 T 36 k 53 1
3 D 20 U 37 l 54 2
4 E 21 V 38 m 55 3
5 F 22 W 39 n 56 4
6 G 23 X 40 o 57 5
7 H 24 Y 41 p 58 6
8 I 25 Z 42 q 59 7
9 J 26 a 43 r 60 8
10 K 27 b 44 s 61 9
11 L 28 c 45 t 62 +
12 M 29 d 46 u 63 /
13 N 30 e 47 v
14 O 31 f 48 w (pad) =
15 P 32 g 49 x
16 Q 33 h 50 y
Special processing is performed if fewer than 24 bits are available
at the end of the data being encoded. A full encoding quantum is
always completed at the end of a quantity. When fewer than 24 input
bits are available in an input group, zero bits are added (on the
right) to form an integral number of 6-bit groups. Padding at the
end of the data is performed using the '=' character.
Since all base64 input is an integral number of octets, only the
-------------------------------------------------
following cases can arise:
(1) the final quantum of encoding input is an integral
multiple of 24 bits; here, the final unit of encoded
output will be an integral multiple of 4 characters
with no "=" padding,
(2) the final quantum of encoding input is exactly 8 bits;
here, the final unit of encoded output will be two
characters followed by two "=" padding characters, or
(3) the final quantum of encoding input is exactly 16 bits;
here, the final unit of encoded output will be three
characters followed by one "=" padding character.
*/
int
b64_ntop(u_char const *src, size_t srclength, char *target, size_t targsize) {
size_t datalength = 0;
u_char input[3];
u_char output[4];
size_t i;
while (2 < srclength) {
input[0] = *src++;
input[1] = *src++;
input[2] = *src++;
srclength -= 3;
output[0] = input[0] >> 2;
output[1] = ((input[0] & 0x03) << 4) + (input[1] >> 4);
output[2] = ((input[1] & 0x0f) << 2) + (input[2] >> 6);
output[3] = input[2] & 0x3f;
Assert(output[0] < 64);
Assert(output[1] < 64);
Assert(output[2] < 64);
Assert(output[3] < 64);
if (datalength + 4 > targsize)
return (-1);
target[datalength++] = Base64[output[0]];
target[datalength++] = Base64[output[1]];
target[datalength++] = Base64[output[2]];
target[datalength++] = Base64[output[3]];
}
/* Now we worry about padding. */
if (0 != srclength) {
/* Get what's left. */
input[0] = input[1] = input[2] = '\0';
for (i = 0; i < srclength; i++)
input[i] = *src++;
output[0] = input[0] >> 2;
output[1] = ((input[0] & 0x03) << 4) + (input[1] >> 4);
output[2] = ((input[1] & 0x0f) << 2) + (input[2] >> 6);
Assert(output[0] < 64);
Assert(output[1] < 64);
Assert(output[2] < 64);
if (datalength + 4 > targsize)
return (-1);
target[datalength++] = Base64[output[0]];
target[datalength++] = Base64[output[1]];
if (srclength == 1)
target[datalength++] = Pad64;
else
target[datalength++] = Base64[output[2]];
target[datalength++] = Pad64;
}
if (datalength >= targsize)
return (-1);
target[datalength] = '\0'; /* Returned value doesn't count \0. */
return (datalength);
}
libresolv_hidden_def (b64_ntop)
/* skips all whitespace anywhere.
converts characters, four at a time, starting at (or after)
src from base - 64 numbers into three 8 bit bytes in the target area.
it returns the number of data bytes stored at the target, or -1 on error.
*/
int
b64_pton (char const *src, u_char *target, size_t targsize)
{
int tarindex, state, ch;
char *pos;
state = 0;
tarindex = 0;
while ((ch = *src++) != '\0') {
if (isspace(ch)) /* Skip whitespace anywhere. */
continue;
if (ch == Pad64)
break;
pos = strchr(Base64, ch);
if (pos == 0) /* A non-base64 character. */
return (-1);
switch (state) {
case 0:
if (target) {
if ((size_t)tarindex >= targsize)
return (-1);
target[tarindex] = (pos - Base64) << 2;
}
state = 1;
break;
case 1:
if (target) {
if ((size_t)tarindex + 1 >= targsize)
return (-1);
target[tarindex] |= (pos - Base64) >> 4;
target[tarindex+1] = ((pos - Base64) & 0x0f)
<< 4 ;
}
tarindex++;
state = 2;
break;
case 2:
if (target) {
if ((size_t)tarindex + 1 >= targsize)
return (-1);
target[tarindex] |= (pos - Base64) >> 2;
target[tarindex+1] = ((pos - Base64) & 0x03)
<< 6;
}
tarindex++;
state = 3;
break;
case 3:
if (target) {
if ((size_t)tarindex >= targsize)
return (-1);
target[tarindex] |= (pos - Base64);
}
tarindex++;
state = 0;
break;
default:
abort();
}
}
/*
* We are done decoding Base-64 chars. Let's see if we ended
* on a byte boundary, and/or with erroneous trailing characters.
*/
if (ch == Pad64) { /* We got a pad char. */
ch = *src++; /* Skip it, get next. */
switch (state) {
case 0: /* Invalid = in first position */
case 1: /* Invalid = in second position */
return (-1);
case 2: /* Valid, means one byte of info */
/* Skip any number of spaces. */
for ((void)NULL; ch != '\0'; ch = *src++)
if (!isspace(ch))
break;
/* Make sure there is another trailing = sign. */
if (ch != Pad64)
return (-1);
ch = *src++; /* Skip the = */
/* Fall through to "single trailing =" case. */
/* FALLTHROUGH */
case 3: /* Valid, means two bytes of info */
/*
* We know this char is an =. Is there anything but
* whitespace after it?
*/
for ((void)NULL; ch != '\0'; ch = *src++)
if (!isspace(ch))
return (-1);
/*
* Now make sure for cases 2 and 3 that the "extra"
* bits that slopped past the last full byte were
* zeros. If we don't check them, they become a
* subliminal channel.
*/
if (target && target[tarindex] != 0)
return (-1);
}
} else {
/*
* We ended by seeing the end of the string. Make sure we
* have no partial bytes lying around.
*/
if (state != 0)
return (-1);
}
return (tarindex);
}

36
tjp/core/bases/_tests/Base16.cpp Normal file
View File

@@ -0,0 +1,36 @@
// TJP COPYRIGHT HEADER
#include <tjp/core/testing/catch.hpp>
#include <tjp/core/bases/Base64.h>
namespace tjp::core {
namespace {
SCENARIO("base16")
{
GIVEN("size calculations")
{
std::vector<char> x;
x.reserve(512);
for (auto i=0; i<512; i++)
{
x.resize(i);
for (auto j=0; j<i; ++j)
x[j] = j*3;
auto e = base64EncodeSize(i);
auto enc = toBase64(x);
REQUIRE(enc.size() == e);
auto d = base64DecodeSize(e);
auto dec = fromBase64(enc);
REQUIRE(d == i);
REQUIRE(dec == x);
}
}
}
} // namespace
} // namespace

58
tjp/core/bases/_tests/Base32.cpp Normal file
View File

@@ -0,0 +1,58 @@
// TJP COPYRIGHT HEADER
#include <tjp/core/testing/catch.hpp>
#include <tjp/core/bases/Base32.h>
namespace tjp::core {
namespace {
SCENARIO("base32")
{
GIVEN("size calculations")
{
std::vector<char> x;
x.reserve(512);
for (auto i=0; i<512; i++)
{
x.resize(i);
for (auto j=0; j<i; ++j)
x[j] = i+j*3;
auto e = base32EncodeSize(i);
auto enc = toBase32(x);
REQUIRE(enc.size() == e);
auto d = base32DecodeSize(enc.data(), e);
REQUIRE(d == i);
auto dec = fromBase32(enc);
REQUIRE(d == i);
auto equals = dec == x;
REQUIRE(equals);
}
}
GIVEN("size u32")
{
auto encodeSize1 = base32EncodeSize(1);
REQUIRE(encodeSize1 < 16);
auto encodeSize2 = base32EncodeSize(2);
REQUIRE(encodeSize2 < 16);
auto encodeSize4 = base32EncodeSize(4);
REQUIRE(encodeSize4 < 16);
auto encodeSize6 = base32EncodeSize(6);
REQUIRE(encodeSize6 < 16);
auto encodeSize8 = base32EncodeSize(8);
REQUIRE(encodeSize8 < 16);
}
}
} // namespace
} // namespace

36
tjp/core/bases/_tests/Base64.cpp Normal file
View File

@@ -0,0 +1,36 @@
// TJP COPYRIGHT HEADER
#include <tjp/core/testing/catch.hpp>
#include <tjp/core/bases/Base64.h>
namespace tjp::core {
namespace {
SCENARIO("base64")
{
GIVEN("size calculations")
{
std::vector<char> x;
x.reserve(512);
for (auto i=0; i<512; i++)
{
x.resize(i);
for (auto j=0; j<i; ++j)
x[j] = j*3;
auto e = base64EncodeSize(i);
auto enc = toBase64(x);
REQUIRE(enc.size() == e);
auto d = base64DecodeSize(e);
auto dec = fromBase64(enc);
REQUIRE(d == i);
REQUIRE(dec == x);
}
}
}
} // namespace
} // namespace

588
tjp/core/compression/_delete/zlib.cpp Normal file
View File

@@ -0,0 +1,588 @@
// TJP COPYRIGHT HEADER
#include "zlib.hpp"
#include <tjp/core/threads/Lock.hpp>
#include <tjp/core/containers/Map.hpp>
#include <tjp/core/algorithm/mem_copy.hpp>
#include <zlib/zlib.h>
namespace tjp::core::compression::zlib {
struct CompressContexts {
Mutex mutex;
Map<int, Vector<void *>> contextsByLevel;
~CompressContexts()
{
for (auto &[l, contexts] : contextsByLevel)
for (auto *c : contexts)
free((z_streamp)c);
}
void *getContext(int level)
{
{
auto lock = lock_of(mutex);
auto &contexts = contextsByLevel[level];
if (!contexts.empty())
{
auto *v = contexts.back();
contexts.pop_back();
return v;
}
}
auto z = (z_streamp)calloc(1, sizeof(z_stream));
deflateInit(z, level);
return z;
}
void putContext(int level, void *v)
{
auto lock = lock_of(mutex);
contextsByLevel[level].push_back(v);
}
} ;
struct DecompressContexts
{
Mutex mutex;
Vector<void *> contexts;
~DecompressContexts()
{
for (auto *c : contexts)
free((z_streamp)c);
}
void *getContext()
{
{
auto lock = lock_of(mutex);
if (!contexts.empty())
{
auto *v = contexts.back();
contexts.pop_back();
return v;
}
}
auto z = (z_streamp)calloc(1, sizeof(z_stream));
inflateInit(z);
return z;
}
void putContext(void *v)
{
auto lock = lock_of(mutex);
contexts.push_back(v);
}
} ;
CompressContexts compressContexts;
DecompressContexts decompressContexts;
// --------------
bool clearStream(void *p, bool inflate)
{
auto z = (z_streamp)p;
if (inflate)
return (inflateReset(z) == Z_OK);
else
return (deflateReset(z) == Z_OK);
}
bool setDict(void *p, bool inflate, char *dict, size_t size)
{
auto z = (z_streamp)p;
int error = 0;
if (inflate)
error = inflateSetDictionary(z, (const Bytef *)dict, (uInt)size);
else
error = deflateSetDictionary(z, (const Bytef *)dict, (uInt)size);
return error == Z_OK;
}
bool getDict(void *p, bool inflate, char *dict, size_t size)
{
auto z = (z_streamp)p;
auto size_ = (uInt)size;
char store[32768];
int error = 0;
if (inflate)
error = inflateGetDictionary(z, (Bytef *)&store[0], &size_);
else
error = deflateGetDictionary(z, (Bytef *)&store[0], &size_);
// zlib seems to store from end to front
auto offset =
size_ > size ?
size_ - size :
0;
mem_copy(dict, &store[offset], size);
return error == Z_OK;
}
// --------------
CompressUnordered::CompressUnordered(int level_) :
level(level_)
{
}
size_t CompressUnordered::execute(char *dest_, size_t destLen_, const char *source_, size_t sourceLen_)
{
const Bytef *source = (const Bytef *)source_;
uLongf sourceLen = sourceLen_;
Bytef *dest = (Bytef *)dest_;
uLongf destLen = destLen_;
auto context = (z_streamp)compressContexts.getContext(level);
context->next_in = (Bytef *)source;
context->avail_in = (uInt)sourceLen;
context->next_out = dest;
context->avail_out = (uInt)destLen;
auto result = deflate(context, Z_FINISH);
auto toSize = context->total_out;
compressContexts.putContext(level, context);
if (result != Z_STREAM_END)
return 0;
return toSize;
}
size_t UncompressUnordered::execute(char *dest_, size_t destLen_, const char *source_, size_t sourceLen_)
{
const Bytef *source = (const Bytef *)source_;
uLongf sourceLen = sourceLen_;
Bytef *dest = (Bytef *)dest_;
uLongf destLen = destLen_;
auto context = (z_streamp)decompressContexts.getContext();
context->next_in = (Bytef *)source;
context->avail_in = (uInt)sourceLen;
context->next_out = dest;
context->avail_out = (uInt)destLen;
auto result = inflate(context, Z_FINISH);
auto toSize = context->total_out;
decompressContexts.putContext(context);
if (result != Z_STREAM_END)
return 0;
return toSize;
}
// --------
CompressOrdered::CompressOrdered(int level_) :
level(level_)
{
memset(dictionary, 0, dictionarySize);
}
CompressOrdered::~CompressOrdered()
{
}
size_t CompressOrdered::execute(char *dest_, size_t destLen_, const char *source_, size_t sourceLen_)
{
const Bytef *source = (const Bytef *)source_;
uLongf sourceLen = sourceLen_;
Bytef *dest = (Bytef *)dest_;
uLongf destLen = destLen_;
auto context = (z_streamp)compressContexts.getContext(level);
if (!clearStream(context, false))
return 0;
if (!setDict(context, false, dictionary, dictionarySize))
return 0;
context->next_in = (Bytef *)source;
context->avail_in = (uInt)sourceLen;
context->next_out = dest;
context->avail_out = (uInt)destLen;
auto result = deflate(context, Z_FINISH);
auto toSize = context->total_out;
if (result == Z_STREAM_END)
getDict(context, false, dictionary, dictionarySize);
compressContexts.putContext(level, context);
if (result != Z_STREAM_END)
return 0;
return toSize;
}
size_t CompressOrdered::execute(Vector<char> &dest, const containers::MemorySegment<char> &source)
{
auto context = (z_streamp)compressContexts.getContext(level);
if (!clearStream(context, false))
return 0;
if (!setDict(context, false, dictionary, dictionarySize))
return 0;
context->next_in = (Bytef *)source.data();
context->avail_in = (uInt)source.size();
auto sizeOffset = dest.size();
dest.resize(dest.size() + sizeof(u32));
auto bos = dest.size();
int result = 0;
while(1)
{
auto newSize = std::max(bos + context->total_out + 2048, (size_t)16384);
dest.resize(newSize);
auto eos = bos + context->total_out;
context->next_out = (Bytef *)dest.data() + eos;
context->avail_out = (uInt)dest.size() - eos;
result = deflate(context, Z_FINISH);
if (result == Z_STREAM_END)
break;
if (result <= Z_ERRNO)
break;
}
getDict(context, false, dictionary, dictionarySize);
u32 toSize = context->total_out;
mem_copy(dest.data() + sizeOffset, (char *)&toSize, sizeof(u32));
auto eos = bos + context->total_out;
dest.resize(eos);
compressContexts.putContext(level, context);
if (result != Z_STREAM_END)
return 0;
return toSize;
}
UncompressOrdered::UncompressOrdered()
{
memset(dictionary, 0, dictionarySize);
}
UncompressOrdered::~UncompressOrdered()
{
}
size_t UncompressOrdered::execute(char *dest_, size_t destLen_, const char *source_, size_t sourceLen_)
{
const Bytef *source = (const Bytef *)source_;
uLongf sourceLen = sourceLen_;
Bytef *dest = (Bytef *)dest_;
uLongf destLen = destLen_;
auto context = (z_streamp)decompressContexts.getContext();
if (!clearStream(context, true))
return 0;
context->next_in = (Bytef *)source;
context->avail_in = (uInt)sourceLen;
context->next_out = dest;
context->avail_out = (uInt)destLen;
auto result = inflate(context, Z_FINISH);
if (result == Z_NEED_DICT)
{
if (!setDict(context, true, dictionary, dictionarySize))
return 0;
result = inflate(context, Z_FINISH);
}
auto toSize = context->total_out;
if (result == Z_STREAM_END)
getDict(context, true, dictionary, dictionarySize);
decompressContexts.putContext(context);
if (result != Z_STREAM_END)
return 0;
return toSize;
}
size_t UncompressOrdered::execute(const containers::MemorySegment<char> &dest, const containers::MemorySegment<char> &source)
{
u32 size = 0;
mem_copy((char *)&size, source.data(), sizeof(u32));
auto dataOffset = sizeof(u32);
auto context = (z_streamp)decompressContexts.getContext();
if (!clearStream(context, true))
return 0;
context->next_in = (Bytef *)source.data() + dataOffset;
context->avail_in = size;
int result = 0;
while(1)
{
context->next_out = (Bytef *)dest.data() + context->total_out;
context->avail_out = (uInt)dest.size() - context->total_out;
result = inflate(context, Z_FINISH);
if (result == Z_NEED_DICT)
{
if (!setDict(context, true, dictionary, dictionarySize))
return 0;
result = inflate(context, Z_FINISH);
}
if (result == Z_STREAM_END)
break;
if (result <= Z_ERRNO)
break;
}
auto toSize = context->total_out;
debug_assert(toSize == dest.size());
getDict(context, true, dictionary, dictionarySize);
decompressContexts.putContext(context);
if (result != Z_STREAM_END)
return 0;
return size + sizeof(32);
}
size_t UncompressOrdered::execute(Vector<char> &dest, const containers::MemorySegment<char> &source)
{
u32 size = 0;
mem_copy((char *)&size, source.data(), sizeof(u32));
auto dataOffset = sizeof(u32);
auto context = (z_streamp)decompressContexts.getContext();
if (!clearStream(context, true))
return 0;
context->next_in = (Bytef *)source.data() + dataOffset;
context->avail_in = size;
auto offset = 0;
int result = 0;
while(1)
{
auto newSize = std::max(offset + context->total_out + 2048, (size_t)16384);
dest.resize(newSize);
context->next_out = (Bytef *)dest.data() + offset + context->total_out;
context->avail_out = (uInt)dest.size() - offset - context->total_out;
result = inflate(context, Z_FINISH);
if (result == Z_NEED_DICT)
{
if (!setDict(context, true, dictionary, DictionarySize))
return 0;
result = inflate(context, Z_FINISH);
}
if (result == Z_STREAM_END)
break;
if (result <= Z_ERRNO)
break;
}
auto toSize = context->total_out;
dest.resize(toSize);
getDict(context, true, dictionary, dictionarySize);
decompressContexts.putContext(context);
if (result != Z_STREAM_END)
return 0;
return size + sizeof(32);
}
// ---------
CompressStream::CompressStream()
{
memset(dictionary, 0, dictionarySize);
}
size_t CompressStream::with(Vector<char> &v, Function<void (CompressStream &)> &&f)
{
auto context = (z_streamp)decompressContexts.getContext();
if (!clearStream(context, true))
return 0;
if (!setDict(context, true, dictionary, dictionarySize))
return 0;
i = (I *)context;
v_ = &v;
p = v.size();
auto initialSize = v.size();
f(*this);
v.resize(p);
auto finalSize = v.size();
decompressContexts.putContext(context);
return finalSize - initialSize;
}
size_t CompressStream::write(const containers::MemorySegment<char> &s)
{
auto &v = *v_;
auto context = (z_streamp)i;
context->next_in = (Bytef *)s.data();
context->avail_in = (uInt)s.size();
// beginning of stream
auto bos = p;
int result = 0;
while(1)
{
auto newSize = std::max(bos + context->total_out + 2048, (size_t)16384);
v.resize(newSize);
context->next_out = (Bytef *)v.data() + bos + context->total_out;
context->avail_out = (uInt)v.size() - bos - context->total_out;
result = deflate(context, Z_FINISH);
if (result == Z_STREAM_END)
break;
if (result <= Z_ERRNO)
break;
}
auto toSize = context->total_out;
p += toSize;
return toSize;
}
DecompressStream::DecompressStream()
{
memset(dictionary, 0, dictionarySize);
}
size_t DecompressStream::with(const containers::MemorySegment<char> &v, Function<void (DecompressStream &)> &&f)
{
auto context = (z_streamp)decompressContexts.getContext();
if (!clearStream(context, true))
return 0;
if (!setDict(context, true, dictionary, dictionarySize))
return 0;
i = (I *)context;
context->next_in = (Bytef *)v.data();
context->avail_in = (uInt)v.size();
f(*this);
auto sizeRead = context->total_in;
decompressContexts.putContext(context);
return sizeRead;
}
size_t DecompressStream::read(const containers::MemorySegment<char> &s)
{
// beginning of stream
int result = 0;
auto context = (z_streamp)i;
context->next_out = (Bytef *)s.data();
context->avail_out = (uInt)s.size();
while(1)
{
result = inflate(context, Z_FINISH);
if (result == Z_NEED_DICT)
{
if (!setDict(context, true, dictionary, DictionarySize))
return 0;
result = inflate(context, Z_FINISH);
}
if (result == Z_STREAM_END)
break;
if (result <= Z_ERRNO)
break;
}
auto toSize = context->total_out;
return toSize;
}
} // namespace

88
tjp/core/compression/_delete/zlib.hpp Normal file
View File

@@ -0,0 +1,88 @@
// TJP COPYRIGHT HEADER
#pragma once
#include "zlib.h"
#include <tjp/core/containers/Vector.h>
#include <tjp/core/containers/MemorySegment.hpp>
#include <tjp/core/containers/Function.hpp>
namespace tjp::core::compression::zlib {
constexpr size_t DictionarySize = 256;
using Dictionary = char[DictionarySize];
struct CompressUnordered
{
static constexpr auto dictionarySize = DictionarySize;
CompressUnordered(int level);
int level;
size_t execute(char *dest, size_t destLen, const char *source, size_t sourceLen);
};
struct UncompressUnordered
{
size_t execute(char *dest, size_t destLen, const char *source, size_t sourceLen);
} ;
struct CompressOrdered {
static constexpr auto dictionarySize = DictionarySize;
CompressOrdered(int level);
~CompressOrdered();
int level;
Dictionary dictionary;
size_t execute(char *dest, size_t destLen, const char *source, size_t sourceLen);
size_t execute(Vector<char> &, const containers::MemorySegment<char> &);
};
struct UncompressOrdered
{
static constexpr auto dictionarySize = DictionarySize;
UncompressOrdered();
~UncompressOrdered();
Dictionary dictionary;
size_t execute(char *dest, size_t destLen, const char *source, size_t sourceLen);
size_t execute(const containers::MemorySegment<char> &, const containers::MemorySegment<char> &);
size_t execute(Vector<char> &, const containers::MemorySegment<char> &);
} ;
struct CompressStream
{
static constexpr auto dictionarySize = DictionarySize;
Dictionary dictionary;
using I = void;
I *i;
Vector<char> *v_;
size_t p;
CompressStream();
size_t with(Vector<char> &, Function<void(CompressStream &)> &&f);
size_t write(const containers::MemorySegment<char> &);
} ;
struct DecompressStream
{
static constexpr auto dictionarySize = DictionarySize;
Dictionary dictionary;
using I = void;
I *i;
DecompressStream();
size_t with(const containers::MemorySegment<char> &, Function<void(DecompressStream &)> &&f);
size_t read(const containers::MemorySegment<char> &);
} ;
} // namespace

282
tjp/core/compression/_tests/zlib.cpp Normal file
View File

@@ -0,0 +1,282 @@
// TJP COPYRIGHT HEADER
#include <tjp/core/testing/catch.hpp>
#include <tjp/core/compression/zlib.hpp>
#include <tjp/core/hash/Hash.h>
#include <tjp/core/types/Types.h>
namespace tjp::core::compression::zlib {
namespace {
SCENARIO("tjp::core::compression::zlib")
{
GIVEN("read and write")
{
CompressStream out(9);
Vector<char> packet;
struct Values {
s16 a;
Vector<int> b;
u32 c;
Vector<s16> d;
};
Values expected {
.a = 42,
.b = { 1, 2, 3, 4, 5 },
.c = 13
};
auto largeNumber = 235324;
expected.d.assign(largeNumber, 12345);
out.with(packet, [&](auto &s) {
s.value(expected.a);
s.vector(expected.b);
s.value(expected.c);
s.vector(expected.d);
});
Values expectedB {
.a = 124,
.b = { 5, 2, 3, 4, 5 },
.c = 23
};
expectedB.d.assign(largeNumber, 6666);
out.with(packet, [&](auto &s) {
s.value(expectedB.a);
s.vector(expectedB.b);
s.value(expectedB.c);
s.vector(expectedB.d);
});
Values real;
real.b.resize(5);
real.d.resize(largeNumber);
DecompressStream in;
auto segment = containers::memory_segment_vector<char>(packet);
in.with(segment, [&](auto &s) {
s.value(real.a);
s.vector(real.b);
s.value(real.c);
s.vector(real.d);
});
REQUIRE(real.a == expected.a);
REQUIRE(real.b == expected.b);
REQUIRE(real.c == expected.c);
auto d_hash = hash_of<u64>(real.d);
auto d_expectedHash = hash_of<u64>(expected.d);
REQUIRE(d_hash == d_expectedHash);
Values realB;
realB.b.resize(5);
realB.d.resize(largeNumber);
in.with(segment, [&](auto &s) {
s.value(realB.a);
s.vector(realB.b);
s.value(realB.c);
s.vector(realB.d);
});
REQUIRE(realB.a == expectedB.a);
REQUIRE(realB.b == expectedB.b);
REQUIRE(realB.c == expectedB.c);
auto bd_hash = hash_of<u64>(realB.d);
auto bd_expectedHash = hash_of<u64>(expectedB.d);
REQUIRE(bd_hash == bd_expectedHash);
}
GIVEN("back-to-back streams")
{
CompressStream out(9);
Vector<char> packet;
Vector<s16> expectedA;
Vector<s16> expectedB;
expectedA.assign(32768, 1111);
expectedB.assign(32768, 2222);
auto firstCompressedSize = out.with(packet, [&](auto &s) {
s.vector(expectedA);
});
auto secondCompressedSize = out.with(packet, [&](auto &s) {
s.vector(expectedB);
});
REQUIRE(firstCompressedSize > 0);
REQUIRE(secondCompressedSize > 0);
DecompressStream in;
auto segment = containers::memory_segment_vector<char>(packet);
Vector<s16> realA;
realA.resize(expectedA.size());
in.with(segment, [&](auto &s) {
s.vector(realA);
});
auto a_hash = hash_of<u64>(realA);
auto a_expectedHash = hash_of<u64>(expectedA);
REQUIRE(a_hash == a_expectedHash);
REQUIRE(segment.size() == secondCompressedSize);
Vector<s16> realB;
realB.resize(expectedB.size());
in.with(segment, [&](auto &s) {
s.vector(realB);
});
auto b_hash = hash_of<u64>(realB);
auto b_expectedHash = hash_of<u64>(expectedB);
REQUIRE(b_hash == b_expectedHash);
REQUIRE(segment.size() == 0);
}
GIVEN("gzip read and write")
{
CompressStream out(9, Format::Gzip);
Vector<char> packet;
struct Values {
s16 a;
Vector<int> b;
u32 c;
Vector<s16> d;
};
Values expected {
.a = 17,
.b = { 9, 8, 7, 6, 5 },
.c = 99
};
auto largeNumber = 32768;
expected.d.assign(largeNumber, 4321);
Values expectedB {
.a = 18,
.b = { 1, 3, 5, 7, 9 },
.c = 100
};
expectedB.d.assign(largeNumber, 8765);
out.with(packet, [&](auto &s) {
s.value(expected.a);
s.vector(expected.b);
s.value(expected.c);
s.vector(expected.d);
});
out.with(packet, [&](auto &s) {
s.value(expectedB.a);
s.vector(expectedB.b);
s.value(expectedB.c);
s.vector(expectedB.d);
});
REQUIRE(packet.size() >= 2);
REQUIRE((unsigned char)packet[0] == 0x1f);
REQUIRE((unsigned char)packet[1] == 0x8b);
Values real;
real.b.resize(expected.b.size());
real.d.resize(largeNumber);
DecompressStream in;
auto segment = containers::memory_segment_vector<char>(packet);
in.with(segment, [&](auto &s) {
s.value(real.a);
s.vector(real.b);
s.value(real.c);
s.vector(real.d);
});
REQUIRE(real.a == expected.a);
REQUIRE(real.b == expected.b);
REQUIRE(real.c == expected.c);
auto d_hash = hash_of<u64>(real.d);
auto d_expectedHash = hash_of<u64>(expected.d);
REQUIRE(d_hash == d_expectedHash);
Values realB;
realB.b.resize(expectedB.b.size());
realB.d.resize(largeNumber);
in.with(segment, [&](auto &s) {
s.value(realB.a);
s.vector(realB.b);
s.value(realB.c);
s.vector(realB.d);
});
REQUIRE(realB.a == expectedB.a);
REQUIRE(realB.b == expectedB.b);
REQUIRE(realB.c == expectedB.c);
auto bd_hash = hash_of<u64>(realB.d);
auto bd_expectedHash = hash_of<u64>(expectedB.d);
REQUIRE(bd_hash == bd_expectedHash);
REQUIRE(segment.size() == 0);
}
GIVEN("gzip back-to-back streams")
{
CompressStream out(9, Format::Gzip);
Vector<char> packet;
Vector<s16> expectedA;
Vector<s16> expectedB;
expectedA.assign(32768, 1111);
expectedB.assign(32768, 2222);
auto firstCompressedSize = out.with(packet, [&](auto &s) {
s.vector(expectedA);
});
auto secondCompressedSize = out.with(packet, [&](auto &s) {
s.vector(expectedB);
});
REQUIRE(firstCompressedSize > 0);
REQUIRE(secondCompressedSize > 0);
REQUIRE(packet.size() >= 2);
REQUIRE((unsigned char)packet[0] == 0x1f);
REQUIRE((unsigned char)packet[1] == 0x8b);
DecompressStream in;
auto segment = containers::memory_segment_vector<char>(packet);
Vector<s16> realA;
realA.resize(expectedA.size());
in.with(segment, [&](auto &s) {
s.vector(realA);
});
auto a_hash = hash_of<u64>(realA);
auto a_expectedHash = hash_of<u64>(expectedA);
REQUIRE(a_hash == a_expectedHash);
REQUIRE(segment.size() == secondCompressedSize);
Vector<s16> realB;
realB.resize(expectedB.size());
in.with(segment, [&](auto &s) {
s.vector(realB);
});
auto b_hash = hash_of<u64>(realB);
auto b_expectedHash = hash_of<u64>(expectedB);
REQUIRE(b_hash == b_expectedHash);
REQUIRE(segment.size() == 0);
}
}
} // namespace
} // namespace

148
tjp/core/compression/_tests/zstd.cpp Normal file
View File

@@ -0,0 +1,148 @@
// TJP COPYRIGHT HEADER
#include <tjp/core/testing/catch.hpp>
#include <tjp/core/compression/zstd.hpp>
#include <tjp/core/hash/Hash.h>
#include <tjp/core/types/Types.h>
namespace tjp::core::compression::zstd {
namespace {
SCENARIO("tjp::core::compression::zstd")
{
GIVEN("read and write")
{
CompressStream out(9);
Vector<char> packet;
struct Values {
s16 a;
Vector<int> b;
u32 c;
Vector<s16> d;
};
Values expected {
.a = 42,
.b = { 1, 2, 3, 4, 5 },
.c = 13
};
auto largeNumber = 235324;
expected.d.assign(largeNumber, 12345);
out.with(packet, [&](auto &s) {
s.value(expected.a);
s.vector(expected.b);
s.value(expected.c);
s.vector(expected.d);
});
Values expectedB {
.a = 124,
.b = { 5, 2, 3, 4, 5 },
.c = 23
};
expectedB.d.assign(largeNumber, 6666);
out.with(packet, [&](auto &s) {
s.value(expectedB.a);
s.vector(expectedB.b);
s.value(expectedB.c);
s.vector(expectedB.d);
});
Values real;
real.b.resize(5);
real.d.resize(largeNumber);
DecompressStream in;
auto segment = containers::memory_segment_vector<char>(packet);
in.with(segment, [&](auto &s) {
s.value(real.a);
s.vector(real.b);
s.value(real.c);
s.vector(real.d);
});
REQUIRE(real.a == expected.a);
REQUIRE(real.b == expected.b);
REQUIRE(real.c == expected.c);
auto d_hash = hash_of<u64>(real.d);
auto d_expectedHash = hash_of<u64>(expected.d);
REQUIRE(d_hash == d_expectedHash);
Values realB;
realB.b.resize(5);
realB.d.resize(largeNumber);
in.with(segment, [&](auto &s) {
s.value(realB.a);
s.vector(realB.b);
s.value(realB.c);
s.vector(realB.d);
});
REQUIRE(realB.a == expectedB.a);
REQUIRE(realB.b == expectedB.b);
REQUIRE(realB.c == expectedB.c);
auto bd_hash = hash_of<u64>(realB.d);
auto bd_expectedHash = hash_of<u64>(expectedB.d);
REQUIRE(bd_hash == bd_expectedHash);
}
GIVEN("back-to-back streams")
{
CompressStream out(9);
Vector<char> packet;
Vector<s16> expectedA;
Vector<s16> expectedB;
expectedA.assign(32768, 1111);
expectedB.assign(32768, 2222);
auto firstCompressedSize = out.with(packet, [&](auto &s) {
s.vector(expectedA);
});
auto secondCompressedSize = out.with(packet, [&](auto &s) {
s.vector(expectedB);
});
REQUIRE(firstCompressedSize > 0);
REQUIRE(secondCompressedSize > 0);
DecompressStream in;
auto segment = containers::memory_segment_vector<char>(packet);
Vector<s16> realA;
realA.resize(expectedA.size());
in.with(segment, [&](auto &s) {
s.vector(realA);
});
auto a_hash = hash_of<u64>(realA);
auto a_expectedHash = hash_of<u64>(expectedA);
REQUIRE(a_hash == a_expectedHash);
REQUIRE(segment.size() == secondCompressedSize);
Vector<s16> realB;
realB.resize(expectedB.size());
in.with(segment, [&](auto &s) {
s.vector(realB);
});
auto b_hash = hash_of<u64>(realB);
auto b_expectedHash = hash_of<u64>(expectedB);
REQUIRE(b_hash == b_expectedHash);
REQUIRE(segment.size() == 0);
}
}
} // namespace
} // namespace

38
tjp/core/compression/remote-run Executable file
View File

@@ -0,0 +1,38 @@
#!/usr/bin/env bash
set -euo pipefail
usage() {
cat <<'EOF'
Usage:
./remote-run <command>
Environment variables:
REMOTE_HOST Default: host.docker.internal
REMOTE_PORT Default: 6666
REMOTE_TOKEN Optional shared token
EOF
}
if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then
usage
exit 0
fi
if [[ $# -lt 1 ]]; then
usage
exit 2
fi
cmd="$1"
host="${REMOTE_HOST:-host.docker.internal}"
port="${REMOTE_PORT:-6666}"
token="${REMOTE_TOKEN:-}"
url="http://${host}:${port}/run/${cmd}"
curl_args=(--silent --show-error --no-buffer "$url")
if [[ -n "$token" ]]; then
curl_args=(-H "X-Remote-Token: ${token}" "${curl_args[@]}")
fi
exec curl "${curl_args[@]}"

525
tjp/core/compression/zlib.cpp Normal file
View File

@@ -0,0 +1,525 @@
// TJP COPYRIGHT HEADER
#include "zlib.hpp"
#include <tjp/core/threads/Lock.hpp>
#include <tjp/core/containers/Map.hpp>
#include <tjp/core/algorithm/mem_copy.hpp>
#include <tjp/core/algorithm/ExecuteOnDestruct.hpp>
#include <zlib/zlib.h>
namespace tjp::core::compression::zlib {
enum class ContextType {
Compress,
Decompress
} ;
namespace {
constexpr int ZlibWindowBits = 15;
constexpr int GzipWindowBits = ZlibWindowBits + 16;
constexpr int InflateAutoWindowBits = ZlibWindowBits + 32;
int window_bits_for(Format format)
{
switch (format)
{
case Format::Zlib:
return ZlibWindowBits;
case Format::Gzip:
return GzipWindowBits;
default:
return ZlibWindowBits;
}
}
size_t compress_key(int level, Format format)
{
return (size_t(level) << 2) | size_t(format);
}
} // namespace
struct CompressContexts {
Mutex mutex;
Map<size_t, Vector<void *>> contextsByKey;
~CompressContexts()
{
for (auto &[k, contexts] : contextsByKey)
for (auto *c : contexts)
free((z_streamp)c);
}
void *getContext(int level, Format format)
{
auto key = compress_key(level, format);
{
auto lock = lock_of(mutex);
auto &contexts = contextsByKey[key];
if (!contexts.empty())
{
auto *v = contexts.back();
contexts.pop_back();
return v;
}
}
auto z = (z_streamp)calloc(1, sizeof(z_stream));
auto result = deflateInit2(
z,
level,
Z_DEFLATED,
window_bits_for(format),
8,
Z_DEFAULT_STRATEGY
);
if (result != Z_OK)
{
free((void *)z);
return nullptr;
}
return z;
}
void putContext(int level, Format format, void *v)
{
auto key = compress_key(level, format);
auto lock = lock_of(mutex);
contextsByKey[key].push_back(v);
}
} ;
struct DecompressContexts
{
Mutex mutex;
Vector<void *> contexts;
~DecompressContexts()
{
for (auto *c : contexts)
free((z_streamp)c);
}
void *getContext()
{
{
auto lock = lock_of(mutex);
if (!contexts.empty())
{
auto *v = contexts.back();
contexts.pop_back();
return v;
}
}
auto z = (z_streamp)calloc(1, sizeof(z_stream));
auto result = inflateInit2(z, InflateAutoWindowBits);
if (result != Z_OK)
{
free((void *)z);
return nullptr;
}
return z;
}
void putContext(void *v)
{
auto lock = lock_of(mutex);
contexts.push_back(v);
}
} ;
CompressContexts compressContexts;
DecompressContexts decompressContexts;
#ifdef TIMPREPSCIUS_ZLIB_USE_DICTIONARY
Dictionary::Dictionary()
{
memset(block, 0, size);
}
bool setDict(void *p, ContextType type, const Dictionary &d)
{
auto z = (z_streamp)p;
int error = 0;
if (type == ContextType::Decompress)
error = inflateSetDictionary(z, (const Bytef *)d.block, (uInt)d.size);
else
error = deflateSetDictionary(z, (const Bytef *)d.block, (uInt)d.size);
return error == Z_OK;
}
bool getDict(void *p, ContextType type, Dictionary &d)
{
auto z = (z_streamp)p;
constexpr auto DictStoreSize = 32768;
char store[DictStoreSize];
memset(store, 0, DictStoreSize);
auto size_ = (uInt)DictStoreSize;
int error = 0;
if (type == ContextType::Decompress)
error = inflateGetDictionary(z, (Bytef *)&store[0], &size_);
else
error = deflateGetDictionary(z, (Bytef *)&store[0], &size_);
// zlib seems to store from end to front
auto offset =
size_ > d.size ?
size_ - d.size :
0;
int firstIndex = -1, lastIndex = -1;
for (auto i=0; i<DictStoreSize; ++i)
{
if (store[i] != 0)
{
if (firstIndex < 0)
firstIndex = i;
lastIndex = i;
}
}
mem_copy(d.block, &store[offset], d.size);
return error == Z_OK;
}
#else
constexpr
bool setDict(void *p, ContextType type, const Dictionary &d) { return true; }
constexpr
bool getDict(void *p, ContextType type, Dictionary &d) { return true; }
#endif
// --------------
bool clearStream(void *p, ContextType type)
{
auto z = (z_streamp)p;
if (type == ContextType::Decompress)
return (inflateReset(z) == Z_OK);
else
return (deflateReset(z) == Z_OK);
}
// --------------
static size_t round_up_to_multiple(size_t number, size_t multiple) {
if (multiple == 0) {
return number; // Avoid division by zero
}
auto remainder = number % multiple;
if (remainder == 0) {
return number; // Already a multiple
}
return number + multiple - remainder;
}
Compressor::Compressor(int level_, Format format_) :
level(level_),
format(format_)
{
}
size_t Compressor::with(V &v, F &&f)
{
error_ = 0;
auto context = (z_streamp)compressContexts.getContext(level, format);
if (!context)
return 0;
auto _ = ExecuteOnDestruct([&]() {
compressContexts.putContext(level, format, context);
i = nullptr;
v_ = nullptr;
});
if (!clearStream(context, ContextType::Compress))
return 0;
if (!setDict(context, ContextType::Compress, dictionary))
return 0;
i = (I *)context;
v_ = &v;
p = v.size();
auto p_ = p;
f(*this);
// finish the stream
while(1)
{
auto newSize = round_up_to_multiple(p + 1, sizeMultiples);
v.resize(newSize);
context->next_out = (Bytef *)v.data() + p;
context->avail_out = uInt(newSize - p);
auto result = deflate(context, Z_FINISH);
p = size_t(context->next_out - (Bytef *)v.data());
if (result == Z_STREAM_END)
break;
if (result <= Z_ERRNO)
{
error_ = result;
break;
}
}
getDict(context, ContextType::Compress, dictionary);
p = size_t(context->next_out - (Bytef *)v.data());
v.resize(p);
return p - p_;
}
size_t Compressor::write(const containers::MemorySegment<char> &s)
{
auto &v = *v_;
auto context = (z_streamp)i;
context->next_in = (Bytef *)s.data();
context->avail_in = (uInt)s.size();
// beginning of stream
int result = 0;
size_t written = 0;
while(1)
{
auto newSize = round_up_to_multiple(p + 1, sizeMultiples);
v.resize(newSize);
context->next_out = (Bytef *)v.data() + p;
context->avail_out = uInt(newSize - p);
result = deflate(context, Z_NO_FLUSH);
p = size_t(context->next_out - (Bytef *)v.data());
written = s.size() - (size_t)context->avail_in;
if (result == Z_STREAM_END)
break;
if (result <= Z_ERRNO)
{
error_ = result;
break;
}
if (context->avail_in == 0)
break;
}
return written;
}
void Compressor::flush()
{
auto &v = *v_;
auto context = (z_streamp)i;
context->next_in = nullptr;
context->avail_in = 0;
// beginning of stream
int result = 0;
while(1)
{
auto newSize = round_up_to_multiple(p + 1, sizeMultiples);
v.resize(newSize);
context->next_out = (Bytef *)v.data() + p;
context->avail_out = uInt(newSize - p);
result = deflate(context, Z_PARTIAL_FLUSH);
p = size_t(context->next_out - (Bytef *)v.data());
if (result == Z_STREAM_END)
break;
if (result <= Z_ERRNO)
{
error_ = result;
break;
}
if (context->avail_out > 0)
break;
}
}
size_t Compressor::size() const
{
return p;
}
bool Compressor::error () const
{
return error_;
}
CompressStream::CompressStream(int level, Format format) :
compressor(level, format)
{
}
size_t CompressStream::with(V &v, F &&f)
{
return compressor.with(v, std::move(f));
}
CompressPacket::CompressPacket(int level_, Format format_) :
level(level_),
format(format_)
{
}
size_t CompressPacket::with(V &v, F &&f)
{
Compressor compressor(level, format);
return compressor.with(v, std::move(f));
}
Decompressor::Decompressor()
{
}
size_t Decompressor::with(V &v, F &&f)
{
error_ = 0;
auto context = (z_streamp)decompressContexts.getContext();
if (!context)
return 0;
i = (I *)context;
v_ = &v;
auto _ = ExecuteOnDestruct([&]() {
decompressContexts.putContext(context);
i = nullptr;
v_ = nullptr;
});
if (!clearStream(context, ContextType::Decompress))
return 0;
context->next_in = (Bytef *)v.data();
context->avail_in = uInt(v.size());
p = 0;
f(*this);
getDict(context, ContextType::Decompress, dictionary);
auto written = context->total_out;
auto consumed = size_t(context->next_in - (Bytef *)v.data());
v.data_ = (char *)context->next_in;
if (consumed > v.size_)
v.size_ = 0;
else
v.size_ -= consumed;
return written;
}
size_t Decompressor::read(const containers::MemorySegment<char> &s)
{
auto &v = *v_;
auto context = (z_streamp)i;
context->next_out = (Bytef *)s.data();
context->avail_out = (uInt)s.size();
int result = 0;
size_t written = 0;
while(1)
{
result = inflate(context, Z_SYNC_FLUSH);
written = size_t(context->next_out - (Bytef *)s.data());
p = size_t(context->next_in - (Bytef *)v.data());
if (result == Z_NEED_DICT)
{
if (!setDict(context, ContextType::Decompress, dictionary))
return 0;
}
if (result == Z_STREAM_END)
break;
if (result <= Z_ERRNO)
{
error_ = result;
break;
}
if (written == s.size())
break;
}
return written;
}
bool Decompressor::eos() const
{
return error() || p >= v_->size();
}
bool Decompressor::error () const
{
return error_;
}
DecompressStream::DecompressStream()
{
}
size_t DecompressStream::with(V &v, F &&f)
{
return decompressor.with(v, std::move(f));
}
DecompressPacket::DecompressPacket()
{
}
size_t DecompressPacket::with(V &v, F &&f)
{
Decompressor decompressor;
return decompressor.with(v, std::move(f));
}
} // namespace

11
tjp/core/compression/zlib.h Normal file
View File

@@ -0,0 +1,11 @@
// TJP COPYRIGHT HEADER
#pragma once
namespace tjp::core::compression::zlib {
struct CompressStream;
struct DecompressStream;
} // namespace

158
tjp/core/compression/zlib.hpp Normal file
View File

@@ -0,0 +1,158 @@
// TJP COPYRIGHT HEADER
#pragma once
#include "zlib.h"
#include <tjp/core/containers/Vector.h>
#include <tjp/core/containers/MemorySegment.hpp>
#include <tjp/core/containers/Function.hpp>
namespace tjp::core::compression::zlib {
enum class Format
{
Zlib,
Gzip
} ;
//#define TIMPREPSCIUS_ZLIB_USE_DICTIONARY
#ifdef TIMPREPSCIUS_ZLIB_USE_DICTIONARY
struct Dictionary
{
static constexpr auto size = 256;
char block[size];
Dictionary();
} ;
#else
struct Dictionary {};
#endif
struct Compressor
{
[[no_unique_address]]
Dictionary dictionary;
static constexpr auto sizeMultiples = 512;
using V = Vector<char>;
using F = Function<void(Compressor &)>;
int level;
Format format;
using I = void;
I *i = nullptr;
V *v_ = nullptr;
size_t p = 0;
int error_ = 0;
Compressor(int level, Format format = Format::Zlib);
size_t size() const;
size_t with(V &, F &&f);
size_t write(const containers::MemorySegment<char> &);
void flush();
template<typename T>
auto value(T &t) {
auto segment = containers::memory_segment_value<char>(t);
return write(segment) == segment.size();
}
template<typename T>
auto vector(T &t) {
auto segment = containers::memory_segment_vector<char>(t);
return write(segment) == segment.size();
}
bool error() const;
} ;
struct CompressStream
{
using V = Compressor::V;
using F = Compressor::F;
Compressor compressor;
CompressStream(int level, Format format = Format::Zlib);
size_t with(V &, F &&f);
} ;
struct CompressPacket
{
using V = Compressor::V;
using F = Compressor::F;
int level;
Format format;
CompressPacket(int level, Format format = Format::Zlib);
size_t with(V &, F &&f);
} ;
struct Decompressor
{
[[no_unique_address]]
Dictionary dictionary;
using V = containers::MemorySegment<char>;
using F = Function<void(Decompressor &)>;
using I = void;
I *i = nullptr;
V *v_ = nullptr;
size_t p = 0;
int error_ = 0;
Decompressor();
bool eos() const;
size_t with(V &, F &&f);
size_t read(const containers::MemorySegment<char> &);
template<typename T>
auto value(T &t) {
auto segment = containers::memory_segment_value<char>(t);
return read(segment) == segment.size();
}
template<typename T>
auto vector(T &t) {
auto segment = containers::memory_segment_vector<char>(t);
return read(segment) == segment.size();
}
bool error () const;
} ;
struct DecompressStream
{
using V = Decompressor::V;
using F = Decompressor::F;
Decompressor decompressor;
DecompressStream();
size_t with(V &, F &&f);
} ;
struct DecompressPacket
{
using V = Decompressor::V;
using F = Decompressor::F;
DecompressPacket();
size_t with(V &, F &&f);
} ;
} // namespace

356
tjp/core/compression/zstd.cpp Normal file
View File

@@ -0,0 +1,356 @@
// TJP COPYRIGHT HEADER
#include "zstd.hpp"
#include <tjp/core/threads/Lock.hpp>
#include <tjp/core/containers/Map.hpp>
#include <tjp/core/algorithm/mem_copy.hpp>
#include <tjp/core/algorithm/ExecuteOnDestruct.hpp>
#include <zstd/zstd.h>
namespace tjp::core::compression::zstd {
enum class ContextType {
Compress,
Decompress
} ;
struct CompressContexts {
Mutex mutex;
Map<int, Vector<void *>> contextsByLevel;
~CompressContexts()
{
for (auto &[l, contexts] : contextsByLevel)
for (auto *c : contexts)
ZSTD_freeCStream((ZSTD_CStream *)c);
}
void *getContext(int level)
{
{
auto lock = lock_of(mutex);
auto &contexts = contextsByLevel[level];
if (!contexts.empty())
{
auto *v = contexts.back();
contexts.pop_back();
return v;
}
}
auto *v = ZSTD_createCStream();
ZSTD_initCStream(v, level);
return v;
}
void putContext(int level, void *v)
{
auto lock = lock_of(mutex);
contextsByLevel[level].push_back(v);
}
} ;
struct DecompressContexts
{
Mutex mutex;
Vector<void *> contexts;
~DecompressContexts()
{
for (auto *c : contexts)
ZSTD_freeDStream((ZSTD_DStream *)c);
}
void *getContext()
{
{
auto lock = lock_of(mutex);
if (!contexts.empty())
{
auto *v = contexts.back();
contexts.pop_back();
return v;
}
}
void *v = ZSTD_createDStream();
return v;
}
void putContext(void *v)
{
auto lock = lock_of(mutex);
contexts.push_back(v);
}
} ;
CompressContexts compressContexts;
DecompressContexts decompressContexts;
// --------------
bool clearStream(void *p, ContextType type)
{
return true;
}
// --------------
static size_t round_up_to_multiple(size_t number, size_t multiple) {
if (multiple == 0) {
return number; // Avoid division by zero
}
auto remainder = number % multiple;
if (remainder == 0) {
return number; // Already a multiple
}
return number + multiple - remainder;
}
Compressor::Compressor(int level_) :
level(level_)
{
}
size_t Compressor::with(V &v, F &&f)
{
auto lock = lock_of(mutex);
auto context = (ZSTD_CStream *)compressContexts.getContext(level);
auto _ = ExecuteOnDestruct([&]() {
compressContexts.putContext(level, context);
i = nullptr;
v_ = nullptr;
});
if (!clearStream(context, ContextType::Compress))
return 0;
i = (I *)context;
v_ = &v;
p = v.size();
auto p_ = p;
f(*this);
// finish the stream
while(1)
{
auto newSize = round_up_to_multiple(p + 1, sizeMultiples);
v.resize(newSize);
ZSTD_outBuffer out;
out.dst = v.data();
out.pos = p;
out.size = v.size();
auto result = ZSTD_endStream(context, &out);
p = out.pos;
if (ZSTD_isError(result))
{
error_ = result;
break;
}
if (result == 0)
break;
}
v.resize(p);
return p - p_;
}
size_t Compressor::write(const containers::MemorySegment<char> &s)
{
auto &v = *v_;
auto context = (ZSTD_CStream *)i;
ZSTD_inBuffer in;
in.size = s.size();
in.src = s.data();
in.pos = 0;
// beginning of stream
size_t result = 0;
while(1)
{
auto newSize = round_up_to_multiple(p + 1, sizeMultiples);
v.resize(newSize);
ZSTD_outBuffer out;
out.dst = v.data();
out.pos = p;
out.size = v.size();
result = ZSTD_compressStream(context, &out, &in);
p = out.pos;
if (ZSTD_isError(result))
{
error_ = result;
break;
}
if (in.pos == in.size)
break;
}
return in.pos;
}
void Compressor::flush()
{
auto &v = *v_;
auto context = (ZSTD_CStream *)i;
// beginning of stream
size_t result = 0;
while(1)
{
auto newSize = round_up_to_multiple(p + 1, sizeMultiples);
v.resize(newSize);
ZSTD_outBuffer out;
out.dst = v.data();
out.pos = p;
out.size = v.size();
result = ZSTD_flushStream(context, &out);
p = out.pos;
if (ZSTD_isError(result))
{
error_ = result;
break;
}
if (out.pos < out.size)
break;
}
}
size_t Compressor::size() const
{
return p;
}
bool Compressor::error () const
{
return error_;
}
CompressPacket::CompressPacket(int level_) :
level(level_)
{
}
size_t CompressPacket::with(V &v, F &&f)
{
Compressor compressor(level);
return compressor.with(v, std::move(f));
}
Decompressor::Decompressor()
{
}
size_t Decompressor::with(V &v, F &&f)
{
auto lock = lock_of(mutex);
auto context = (ZSTD_DCtx *)decompressContexts.getContext();
i = (I *)context;
v_ = &v;
auto _ = ExecuteOnDestruct([&]() {
decompressContexts.putContext(context);
i = nullptr;
v_ = nullptr;
});
if (!clearStream(context, ContextType::Decompress))
return 0;
p = 0;
f(*this);
v.data_ += p;
v.size_ -= p;
return p;
}
size_t Decompressor::read(const containers::MemorySegment<char> &s)
{
auto &v = *v_;
auto context = (ZSTD_DCtx *)i;
ZSTD_inBuffer in;
in.size = v.size();
in.src = v.data();
in.pos = p;
ZSTD_outBuffer out;
out.dst = (void *)s.data();
out.pos = 0;
out.size = s.size();
size_t result = 0;
while(1)
{
result = ZSTD_decompressStream(context, &out, &in);
p = in.pos;
if (ZSTD_isError(result))
{
error_ = result;
break;
}
if (in.pos == in.size)
break;
if (out.pos == out.size)
break;
}
return out.pos;
}
bool Decompressor::eos() const
{
return error() || p >= v_->size();
}
bool Decompressor::error () const
{
return error_;
}
DecompressPacket::DecompressPacket()
{
}
size_t DecompressPacket::with(V &v, F &&f)
{
Decompressor decompressor;
return decompressor.with(v, std::move(f));
}
} // namespace

14
tjp/core/compression/zstd.h Normal file
View File

@@ -0,0 +1,14 @@
// TJP COPYRIGHT HEADER
#pragma once
namespace tjp::core::compression::zstd {
struct CompressPacket;
struct DecompressPacket;
using CompressStream = CompressPacket;
using DecompressStream = DecompressPacket;
} // namespace

111
tjp/core/compression/zstd.hpp Normal file
View File

@@ -0,0 +1,111 @@
// TJP COPYRIGHT HEADER
#pragma once
#include "zstd.h"
#include <tjp/core/containers/Vector.h>
#include <tjp/core/containers/MemorySegment.hpp>
#include <tjp/core/containers/Function.hpp>
#include <tjp/core/threads/TestMutex.hpp>
namespace tjp::core::compression::zstd {
struct Compressor
{
core::TestMutex mutex;
static constexpr auto sizeMultiples = 512;
using V = Vector<char>;
using I = void;
using F = Function<void(Compressor &)>;
int level;
I *i = nullptr;
V *v_ = nullptr;
size_t p = 0;
size_t error_ = 0;
Compressor(int level);
size_t with(V &, F &&f);
size_t size() const;
size_t write(const containers::MemorySegment<char> &);
void flush();
template<typename T>
auto value(const T &t) {
auto segment = containers::memory_segment_value<char>(t);
return write(segment) == segment.size();
}
template<typename T>
auto vector(const T &t) {
auto segment = containers::memory_segment_vector<char>(t);
return write(segment) == segment.size();
}
bool error () const;
} ;
struct CompressPacket
{
using V = Compressor::V;
using F = Compressor::F;
int level;
CompressPacket(int level);
size_t with(V &, F &&f);
} ;
struct Decompressor
{
core::TestMutex mutex;
using V = containers::MemorySegment<char>;
using I = void;
using F = Function<void(Decompressor &)>;
I *i = nullptr;
V *v_ = nullptr;
size_t p = 0;
size_t error_ = 0;
Decompressor();
bool eos() const;
size_t with(V &, F &&f);
size_t read(const containers::MemorySegment<char> &);
template<typename T>
auto value(T &t) {
auto segment = containers::memory_segment_value<char>(t);
return read(segment) == segment.size();
}
template<typename T>
auto vector(T &t) {
auto segment = containers::memory_segment_vector<char>(t);
return read(segment) == segment.size();
}
bool error () const;
};
struct DecompressPacket
{
using V = Decompressor::V;
using F = Decompressor::F;
DecompressPacket();
size_t with(V &, F &&);
} ;
} // namespace

63
tjp/core/containers/Alias.hpp Normal file
View File

@@ -0,0 +1,63 @@
// TJP COPYRIGHT HEADER
#pragma once
namespace tjp::core {
template<typename T>
struct Alias : T {
using Super = T;
using Self = Alias;
template<typename ... Args>
Alias(Args&& ... args) :
Super(std::forward<Args>(args)...)
{}
} ;
#define ALIAS_OF_HEADER(U, ...) \
struct U;
// https://stackoverflow.com/questions/8942912/how-to-pass-multi-argument-templates-to-macros
#define ALIAS_OF(U, ...) \
struct U : __VA_ARGS__ { \
using Super = __VA_ARGS__; \
using Self = U; \
template<typename ... Args> \
U(Args&& ... args) : \
Super(std::forward<Args>(args)...) \
{} \
} \
#define ALIAS_OF_WITH_SIMPLE_ASSIGNMENT(U, ...) \
struct U : __VA_ARGS__ { \
using Super = __VA_ARGS__; \
using Self = U; \
template<typename ... Args> \
U(Args&& ... args) : \
Super(std::forward<Args>(args)...) \
{} \
\
template<typename V, typename = std::enable_if_t<!std::is_same_v<std::decay_t<V>, U> && !std::is_same_v<std::decay_t<V>, Super>>> \
U &operator =(V &&u) = delete; \
\
U &operator =(const Super &u) { Super::operator=(u); return *this; } \
U &operator =(Super &&u) { Super::operator=(std::move(u)); return *this; } \
} \
// U &operator =(const U &u) { Super::operator=(u); return *this; } \
#define ALIAS_OF_WITH_TEMPLATE_ASSIGNMENT(U, ...) \
struct U : __VA_ARGS__ { \
using Super = __VA_ARGS__; \
using Self = U; \
template<typename ... Args> \
U(Args&& ... args) : \
Super(std::forward<Args>(args)...) \
{} \
\
template<typename V> \
auto &operator =(V &&t) { Super::operator=(std::forward<V>(t)); return *this; } \
} \
} // namespace

88
tjp/core/containers/Changed.hpp Normal file
View File

@@ -0,0 +1,88 @@
// TJP COPYRIGHT HEADER
#pragma once
namespace tjp::core {
template<typename T>
struct Changed
{
T value;
bool changed_ = true;
using Self = Changed;
Changed(T value_) :
value(value_)
{
}
Changed():
changed_(false)
{
}
operator T &()
{
return value;
}
void set(const T &rhs)
{
if (rhs == value)
return;
value = rhs;
changed_ = true;
}
Self &operator =(const T &rhs)
{
set(rhs);
return *this;
}
void reset()
{
changed_ = false;
}
void mark()
{
changed_ = true;
}
bool use()
{
bool result = changed_;
changed_ = false;
return result;
}
bool changed() const
{
return changed_;
}
auto *operator ->() const
{
return &value;
}
auto *operator ->()
{
return &value;
}
auto &operator *() const
{
return value;
}
auto &operator *()
{
return value;
}
} ;
} // namespace

198
tjp/core/containers/CircularBuffer.hpp Normal file
View File

@@ -0,0 +1,198 @@
// TJP COPYRIGHT HEADER
#pragma once
#include <tjp/core/assert/debug_assert.h>
#include <tjp/core/containers/Vector.h>
#include <tjp/core/containers/Array.h>
#include <cstring>
namespace tjp::core {
template<typename T>
struct CircularBuffer {
Vector<T> buffer;
size_t begin_ = 0, end_ = 0;
size_t size_ = 0;
size_t begin() const
{
return begin_;
}
size_t end() const
{
return end_;
}
void clear ()
{
size_ = end_ = begin_ = 0;
}
size_t size() const
{
return size_;
}
bool empty() const
{
return size() == 0;
}
size_t available() const
{
return buffer.size() - size();
}
struct Segment {
size_t to, from, size;
} ;
const T *data () const
{
return buffer.data();
}
T *data ()
{
return buffer.data();
}
Array<Segment, 2> segments(size_t from_, size_t dataSize) const
{
auto bufferSize = buffer.size();
if (bufferSize == 0)
{
return {
Segment { 0, 0, 0 },
Segment { 0, 0, 0 }
} ;
}
auto from = from_ % bufferSize;
auto to_0 = from + dataSize;
auto to_1 = (size_t)0;
if (to_0 > bufferSize)
{
to_1 = to_0 % bufferSize;
to_0 = to_0 - to_1;
}
auto seg_0 = (to_0 - from);
auto seg_1 = (size_t)to_1;
return {
Segment { from, 0, seg_0 },
Segment { 0, seg_0, seg_1 }
} ;
}
void grow(size_t dataSize)
{
if (dataSize == 0)
return;
if (available() <= dataSize)
{
auto previousBufferSize = buffer.size();
buffer.resize(previousBufferSize + dataSize);
// if there is data at the beginning of the buffer, we need to
// put some of it at the end
if (end_ <= begin_)
{
auto previousEnd = end_;
end_ = previousBufferSize;
if (previousEnd > 0)
{
size_ -= previousEnd;
write(buffer.data(), previousEnd);
}
}
}
size_ += dataSize;
}
size_t write(const T *data, size_t dataSize)
{
auto size = size_;
grow(dataSize);
auto segments_ = segments(begin_ + size, dataSize);
for (auto &segment: segments_)
{
if (segment.size)
{
debug_assert(segment.to + segment.size <= buffer.size());
if (data)
std::memmove(buffer.data() + segment.to, data + segment.from, segment.size * sizeof(T));
else
std::memset(buffer.data() + segment.to, 0, segment.size * sizeof(T));
end_ = segment.to + segment.size;
end_ = end_ % buffer.size();
}
}
return dataSize;
}
size_t peek(T *data, size_t dataSize) const
{
if (dataSize == 0)
return dataSize;
if (size_ < dataSize)
dataSize = size_;
auto segments_ = segments(begin_, dataSize);
for (auto &segment: segments_)
{
if (segment.size)
{
std::memcpy(data + segment.from, buffer.data() + segment.to, segment.size * sizeof(T));
}
}
return dataSize;
}
size_t read(T *data, size_t dataSize)
{
if (dataSize == 0)
return dataSize;
if (size_ < dataSize)
dataSize = size_;
peek(data, dataSize);
skip(dataSize);
return dataSize;
}
size_t skip(size_t dataSize)
{
if (dataSize == 0)
return dataSize;
debug_assert(size_ >= dataSize);
begin_ += dataSize;
begin_ = begin_ % buffer.size();
size_ -= dataSize;
return dataSize;
}
} ;
} // namespace

42
tjp/core/containers/InvalidatingCache.hpp Normal file
View File

@@ -0,0 +1,42 @@
// TJP COPYRIGHT HEADER
#pragma once
#include <tjp/core/time/Time.hpp>
#include <tjp/core/containers/Optional.hpp>
namespace tjp::core {
template<typename T>
struct InvalidatingCache
{
Optional<T> value;
bool hasValue(time::Time now)
{
return value;
}
template<typename U>
T &cache(U &&u)
{
value.emplace(std::forward<U>(u));
return *value;
}
template<typename F>
T &cacheIf(time::Time now, F &&f)
{
if (!hasValue(now))
return cache(f());
return *value;
}
void invalidate()
{
value.reset();
}
} ;
} // namespace

52
tjp/core/containers/InvalidatingTimedCache.hpp Normal file
View File

@@ -0,0 +1,52 @@
// TJP COPYRIGHT HEADER
#pragma once
#include <tjp/core/time/Time.hpp>
#include <tjp/core/containers/Optional.hpp>
namespace tjp::core {
template<typename T>
struct InvalidatingTimedCache
{
time::Duration invalidAfter = time::Duration::Zero;
Optional<T> value;
Optional<time::Time> invalidatedAt;
bool hasValue(time::Time now)
{
if (!value)
return false;
if (invalidatedAt)
return now < (*invalidatedAt + invalidAfter);
return true;
}
template<typename U>
T &cache(U &&u)
{
value.emplace(std::forward<U>(u));
invalidatedAt.reset();
return *value;
}
template<typename F>
T &cacheIf(time::Time now, F &&f)
{
if (!hasValue(now))
return cache(f());
return *value;
}
void invalidate(time::Time now)
{
invalidatedAt.emplace(now);
}
} ;
} // namespace

101
tjp/core/containers/SizedVector.hpp Normal file
View File

@@ -0,0 +1,101 @@
// TJP COPYRIGHT HEADER
#pragma once
#include <tjp/core/assert/debug_assert.h>
#include <tjp/core/containers/Vector.h>
namespace tjp::core {
template<typename T>
struct SizedVector
{
using V = Vector<T>;
using iterator = typename V::iterator;
using const_iterator = typename V::const_iterator;
using value_type = typename V::value_type;
using reference = typename V::reference;
V v;
size_t size_ = 0;
auto reserved() const { return v.size(); }
void reserve(size_t size)
{
if (size > v.size())
v.resize(size);
}
auto capacity () const {
return v.size();
}
auto resize (size_t size)
{
reserve(size);
return size_ = size;
}
auto &front () { return v.front(); }
auto &front () const { return v.front(); }
auto &back () { return at(size_-1); }
auto &back () const { return at(size_-1); }
auto begin () { return v.begin(); }
auto end() { return v.begin() + size(); };
auto begin () const { return v.begin(); }
auto end() const { return v.begin() + size(); };
auto &at(size_t i)
{
debug_assert(i < size_);
return v[i];
}
auto &at(size_t i) const
{
debug_assert(i < size_);
return v[i];
}
auto &operator[](size_t i) { return at(i); }
auto &operator[](size_t i) const { return at(i); }
auto *data() { return v.data(); }
auto *data() const { return v.data(); }
auto empty() const { return size_ == 0; }
auto size() const { return size_; }
auto push_back(const T &t)
{
debug_assert(size_ < v.size());
if (size_ < v.size())
{
size_++;
back() = t;
}
}
auto pop_back(const T &t)
{
debug_assert(size_ > 0);
if (size_ > 0)
size_--;
}
// can only be used to remove a portion at the end (as of now)
template<typename A, typename B>
auto erase(A &&a, B &&b)
{
debug_assert(b == end());
auto distance = std::distance(a, b);
size_ -= distance;
}
} ;
} // namespace

39
tjp/core/containers/TimedCache.hpp Normal file
View File

@@ -0,0 +1,39 @@
// TJP COPYRIGHT HEADER
#pragma once
#include <tjp/core/time/Time.hpp>
namespace tjp::core {
template<typename T>
struct TimedCache
{
time::Duration expiresAfter;
Optional<T> value;
time::Time cachedAt = time::Time::Zero;
bool hasValue(time::Time now)
{
return value && now < (cachtedAt + expiresAfter);
}
template<typename U>
T &cache(U &&u)
{
value.emplace(std::forward<U>(u));
return *value;
}
template<typename F>
T &cacheIf(time::Time now, F &&f)
{
if (!hasValue(now))
return cache(f());
return *value;
}
} ;
} // namespace

71
tjp/core/containers/_tests/CircularBuffer.cpp Normal file
View File

@@ -0,0 +1,71 @@
// TJP COPYRIGHT HEADER
#include <tjp/core/testing/catch.hpp>
#include <tjp/core/containers/CircularBuffer.hpp>
namespace tjp::core {
namespace {
SCENARIO("CircularBuffer")
{
GIVEN("a circular buffer")
{
typedef int T;
CircularBuffer<T> c;
WHEN("adding in blocks")
{
constexpr int blockSize = 1024;
T removes[blockSize];
T adds[blockSize];
for (auto i=0; i<std::size(adds); ++i)
adds[i] = i;
for (auto i=0; i<64; ++i)
{
c.write(adds, blockSize);
c.read(removes, blockSize);
bool isSame = memcmp(adds, removes, blockSize * sizeof(T)) == 0;
REQUIRE(isSame);
}
for (auto i=0; i<64; ++i)
c.write(adds, blockSize);
for (auto i=0; i<32; ++i)
{
c.read(removes, blockSize);
bool isSame = memcmp(adds, removes, blockSize * sizeof(T)) == 0;
REQUIRE(isSame);
}
for (auto i=0; i<1024; ++i)
{
c.read(removes, blockSize);
c.write(adds, blockSize);
bool isSame = memcmp(adds, removes, blockSize * sizeof(T)) == 0;
REQUIRE(isSame);
}
for (auto i=0; i<32; ++i)
{
c.write(adds, blockSize);
}
for (auto i=0; i<64; ++i)
{
c.read(removes, blockSize);
bool isSame = memcmp(adds, removes, blockSize * sizeof(T)) == 0;
REQUIRE(isSame);
}
}
}
}
} // namespace
} // namespace

168
tjp/core/csv/Csv.cpp Normal file
View File

@@ -0,0 +1,168 @@
// TJP COPYRIGHT HEADER
#include "Csv.hpp"
// Adapted from
// https://stackoverflow.com/questions/1120140/how-can-i-read-and-parse-csv-files-in-c
#include <tjp/core/iterators/enumerate.hpp>
#include <sstream>
namespace tjp {
namespace core {
CSV::CSV(std::istream& str_, bool readHeader_) :
str(str_)
{
if (readHeader_)
readHeader();
}
void CSV::readHeader ()
{
readNextRow();
int i=0;
for (auto &s : m_data)
{
m_header[s] = i;
++i;
}
}
std::string const& CSV::operator[](std::size_t index) const
{
return m_data[index];
}
std::string const& CSV::operator[](const std::string &index) const
{
auto i = m_header.find(index);
return m_data[i->second];
}
std::size_t CSV::size() const
{
return m_data.size();
}
bool CSV::readNextRow()
{
std::string line;
std::getline(str, line);
if (!line.empty() && line.back() == '\r')
line.resize(line.size()-1);
std::stringstream lineStream(line);
std::string cell;
m_data.clear();
while(std::getline(lineStream, cell, ','))
{
m_data.push_back(cell);
}
// This checks for a trailing comma with no data after it.
if (!lineStream && cell.empty())
{
// If there was a trailing comma then add an empty element.
m_data.push_back("");
}
return str.good();
}
FastCSV::FastCSV(std::istream& str_, bool readHeader_) :
str(str_),
lineTokens(Tokenizer::Tokens("\n")),
csvTokens(Tokenizer::Tokens(","))
{
const auto bufferSize = 1024;
char buffer[bufferSize];
while (str_)
{
str_.read(buffer, bufferSize);
data.insert(data.end(), buffer, buffer + str_.gcount());
}
tokenizer = Tokenizer(data.data(), data.size(), lineTokens);
if (readHeader_)
readHeader();
}
void FastCSV::readHeader ()
{
readNextRow();
int i=0;
for (auto &s : m_data)
{
m_header[s] = i;
++i;
}
}
FastCSV::Result FastCSV::operator[](std::size_t index) const
{
return Result(m_data[index]);
}
FastCSV::Segment FastCSV::segment(std::size_t index) const
{
return m_data[index];
};
std::size_t FastCSV::index(const Segment &index) const
{
auto i = m_header.find(index);
if (i == m_header.end())
return -1;
return i->second;
}
FastCSV::Result FastCSV::operator[](const Segment &index) const
{
auto i = m_header.find(index);
return Result(m_data[i->second]);
}
std::size_t FastCSV::size() const
{
return m_data.size();
}
bool FastCSV::readNextRow()
{
Segment line;
char token;
bool result = tokenizer.next(line, token);
if (!result)
return false;
if (!line.empty() && line.back() == '\r')
line.remove_suffix(1);
Tokenizer lineTokenizer(line.data(), line.size(), csvTokens);
Segment cell;
char cellToken;
m_data.clear();
while (lineTokenizer.next(cell, cellToken))
{
m_data.push_back(cell);
}
// This checks for a trailing comma with no data after it.
if (cell.empty())
{
// If there was a trailing comma then add an empty element.
m_data.push_back(Segment());
}
return true;
}
} // namespace
} // namespace

208
tjp/core/csv/Csv.h Normal file
View File

@@ -0,0 +1,208 @@
// TJP COPYRIGHT HEADER
#pragma once
#include <iterator>
#include <iostream>
#include <fstream>
#include <sstream>
#include <vector>
#include <string>
#include <map>
#include <tjp/core/string/Tokenizer.hpp>
// Adapted from
// https://stackoverflow.com/questions/1120140/how-can-i-read-and-parse-csv-files-in-c
#include <tjp/core/iterators/enumerate.hpp>
namespace tjp {
namespace core {
class CSV
{
public:
CSV(std::istream& str_, bool readHeader_ = true) :
str(str_)
{
if (readHeader_)
readHeader();
}
void readHeader ()
{
readNextRow();
int i=0;
for (auto &s : m_data)
{
m_header[s] = i;
++i;
}
}
public:
std::string const& operator[](std::size_t index) const
{
return m_data[index];
}
std::string const& operator[](const std::string &index) const
{
auto i = m_header.find(index);
return m_data[i->second];
}
std::size_t size() const
{
return m_data.size();
}
bool readNextRow()
{
std::string line;
std::getline(str, line);
if (!line.empty() && line.back() == '\r')
line.resize(line.size()-1);
std::stringstream lineStream(line);
std::string cell;
m_data.clear();
while(std::getline(lineStream, cell, ','))
{
m_data.push_back(cell);
}
// This checks for a trailing comma with no data after it.
if (!lineStream && cell.empty())
{
// If there was a trailing comma then add an empty element.
m_data.push_back("");
}
return str.good();
}
protected:
std::istream& str;
std::map<std::string, size_t> m_header;
std::vector<std::string> m_data;
};
class FastCSV
{
protected:
typedef std::string_view Segment;
typedef std::string Result;
std::vector<char> data;
typedef Tokenizer<Segment, TokenVector> Tokenizer;
Tokenizer::Tokens lineTokens;
Tokenizer::Tokens csvTokens;
Tokenizer tokenizer;
public:
FastCSV(std::istream& str_, bool readHeader_ = true) :
str(str_),
lineTokens(Tokenizer::Tokens("\n")),
csvTokens(Tokenizer::Tokens(","))
{
const auto bufferSize = 1024;
char buffer[bufferSize];
while (str_)
{
str_.read(buffer, bufferSize);
data.insert(data.end(), buffer, buffer + str_.gcount());
}
tokenizer = Tokenizer(data.data(), data.size(), lineTokens);
if (readHeader_)
readHeader();
}
void readHeader ()
{
readNextRow();
int i=0;
for (auto &s : m_data)
{
m_header[s] = i;
++i;
}
}
public:
Result operator[](std::size_t index) const
{
return Result(m_data[index]);
}
Segment segment(std::size_t index) const
{
return m_data[index];
};
std::size_t index(const Segment &index) const
{
auto i = m_header.find(index);
if (i == m_header.end())
return -1;
return i->second;
}
Result operator[](const Segment &index) const
{
auto i = m_header.find(index);
return Result(m_data[i->second]);
}
std::size_t size() const
{
return m_data.size();
}
bool readNextRow()
{
Segment line;
char token;
bool result = tokenizer.next(line, token);
if (!result)
return false;
if (!line.empty() && line.back() == '\r')
line.remove_suffix(1);
Tokenizer lineTokenizer(line.data(), line.size(), csvTokens);
Segment cell;
char cellToken;
m_data.clear();
while (lineTokenizer.next(cell, cellToken))
{
m_data.push_back(cell);
}
// This checks for a trailing comma with no data after it.
if (cell.empty())
{
// If there was a trailing comma then add an empty element.
m_data.push_back(Segment());
}
return true;
}
protected:
std::istream& str;
std::map<Segment, size_t> m_header;
std::vector<Segment> m_data;
};
} // namespace
} // namespace

82
tjp/core/csv/Csv.hpp Normal file
View File

@@ -0,0 +1,82 @@
// TJP COPYRIGHT HEADER
#pragma once
#include <iosfwd>
#include <vector>
#include <string>
#include <map>
#include <tjp/core/string/Tokenizer.hpp>
// Adapted from
// https://stackoverflow.com/questions/1120140/how-can-i-read-and-parse-csv-files-in-c
#include <tjp/core/iterators/enumerate.hpp>
namespace tjp {
namespace core {
class CSV
{
public:
CSV(std::istream& str_, bool readHeader_ = true);
void readHeader ();
public:
std::string const& operator[](std::size_t index) const;
std::string const& operator[](const std::string &index) const;
std::size_t size() const;
bool readNextRow();
protected:
std::istream& str;
std::map<std::string, size_t> m_header;
std::vector<std::string> m_data;
};
class FastCSV
{
protected:
typedef std::string_view Segment;
typedef std::string Result;
std::vector<char> data;
typedef core::Tokenizer<Segment, TokenVector> Tokenizer;
Tokenizer::Tokens lineTokens;
Tokenizer::Tokens csvTokens;
Tokenizer tokenizer;
public:
FastCSV(std::istream& str_, bool readHeader_ = true);
void readHeader ();
public:
Result operator[](std::size_t index) const;
Segment segment(std::size_t index) const;
std::size_t index(const Segment &index) const;
Result operator[](const Segment &index) const;
std::size_t size() const;
bool readNextRow();
protected:
std::istream& str;
std::map<Segment, size_t> m_header;
std::vector<Segment> m_data;
};
} // namespace
} // namespace

16
tjp/core/csv/_tests/Csv.cpp Normal file
View File

@@ -0,0 +1,16 @@
// TJP COPYRIGHT HEADER
#include <tjp/core/csv/Csv.h>
using namespace tjp::core;
void compile()
{
std::ifstream file("plop.csv");
CSV csv(file);
while(csv.readNextRow())
{
std::cout << "4th Element(" << csv[3] << ")\n";
}
}

25
tjp/core/delegate/Delegate+Single.h Normal file
View File

@@ -0,0 +1,25 @@
// TJP COPYRIGHT HEADER
#pragma once
#include "Delegate.h"
namespace tjp::core::delegate_single {
template<typename Delegate_>
struct Delegator;
template<typename As_, typename Delegate_>
using SpecializedDelegator = delegate::SpecializedDelegator<As_, Delegate_>;
template<typename Delegate_>
using Token = delegate::Token<Delegate_, Delegator<Delegate_>>;
template<typename Token_>
using WeakToken = delegate::WeakToken<Token_>;
template<typename As_, typename Token_, typename Into_=StrongPtr<As_>>
using TokenTyped = delegate::TokenTyped<As_, Token_, Into_>;
} // namespace

54
tjp/core/delegate/Delegate+Single.hpp Normal file
View File

@@ -0,0 +1,54 @@
// TJP COPYRIGHT HEADER
#pragma once
#include "Delegate+Single.h"
#include "Delegate.hpp"
#include <tjp/core/ptr/Ptr.h>
namespace tjp {
namespace core {
namespace delegate_single {
template<typename Delegate_>
struct Delegator
{
using Delegate = Delegate_;
using TokenType = Token<Delegate>;
Delegate *delegate = nullptr;
virtual TokenType addDelegate(Delegate *delegate) = 0;
virtual void insertDelegate(Delegate *delegate_)
{
debug_assert(delegate == nullptr);
delegate = delegate_;
}
virtual void eraseDelegate(Delegate *delegate_)
{
debug_assert(delegate == delegate_);
delegate = nullptr;
}
template<typename F>
void eachDelegate(F &&f)
{
if (delegate)
f(delegate);
}
} ;
template<typename As, typename Delegator>
using SpecializedDelegator =
delegate::SpecializedDelegator<As, Delegator>;
template<typename As, typename Delegate>
using LockingDelegator =
delegate::LockingDelegatorOf<SpecializedDelegator<As, Delegator<Delegate>>>;
} // namespace
} // namespace
} // namespace

25
tjp/core/delegate/Delegate.h Normal file
View File

@@ -0,0 +1,25 @@
// TJP COPYRIGHT HEADER
#pragma once
#include <tjp/core/ptr/Ptr.h>
namespace tjp::core::delegate {
template<typename Delegate_>
struct Delegator;
template<typename As_, typename Delegate_>
struct SpecializedDelegator;
template<typename Delegate_, typename DelegatorType_= Delegator<Delegate_>>
struct Token;
template<typename Token_>
struct WeakToken;
template<typename As_, typename Token_, typename Into_=StrongPtr<As_>>
struct TokenTyped;
} // namespace

342
tjp/core/delegate/Delegate.hpp Normal file
View File

@@ -0,0 +1,342 @@
// TJP COPYRIGHT HEADER
#pragma once
#include "Delegate.h"
#include <tjp/core/ptr/Ptr.hpp>
#include <tjp/core/containers/List.h>
#include <tjp/core/containers/SafeIteration.hpp>
#include <tjp/core/containers/Function.h>
#include <tjp/core/algorithm/list_erase_value_one.hpp>
#include <any>
namespace tjp {
namespace core {
namespace delegate {
template<typename T>
auto strong_(T &&t)
{
return strong(std::forward<T>(t));
}
template<typename T, typename U>
auto cast_to_strong(const StrongPtr<U> &t)
{
return std::static_pointer_cast<T>(t);
}
template<typename T, typename U>
auto cast_to_strong(const WeakPtr<U> &t)
{
return cast_to_strong<T>(strong(t));
}
template<typename Delegate_, typename DelegatorType_>
struct Token
{
using Delegate = Delegate_;
using DelegatorType = DelegatorType_;
StrongPtr<DelegatorType> delegator;
Token (const Token &) = delete;
Token (Token &&rhs) :
delegator(std::move(rhs.delegator))
{}
Token() {}
Token(const StrongPtr<DelegatorType> &delegator_) :
delegator(delegator_)
{
}
~Token ()
{
debug_assert(delegator == nullptr);
}
Token &operator =(Token &&token_)
{
delegator = std::move(token_.delegator);
return *this;
}
void release(Delegate *delegate)
{
if (!delegator)
return;
delegator->eraseDelegate(delegate);
delegator = nullptr;
}
auto &get () { return delegator; }
auto &get () const { return delegator; }
template<typename T>
auto strong ()
{
return cast_to_strong<T>(delegator);
}
template<typename T>
auto strong () const
{
return cast_to_strong<T>(delegator);
}
template<typename T>
auto ptr ()
{
return static_cast<T *>(ptr_of(delegator));
}
template<typename T>
auto ptr () const
{
return static_cast<T *>(ptr_of(delegator));
}
auto &operator ->() { return get(); }
auto &operator ->() const { return get(); }
operator bool() const
{
return delegator != nullptr;
}
} ;
template<typename Token_>
struct WeakToken
{
using TokenType = Token_;
using Delegate = typename TokenType::Delegate;
using DelegatorType = typename TokenType::DelegatorType;
WeakPtr<DelegatorType> delegator;
WeakToken() {}
WeakToken (const WeakToken &) = delete;
WeakToken (TokenType &&rhs) :
delegator(weak(rhs.delegator))
{
rhs.delegator = nullptr;
}
WeakToken (WeakToken &&rhs) :
delegator(std::move(rhs.delegator))
{
}
WeakToken(const StrongPtr<DelegatorType> &delegator_) :
delegator(weak(delegator_))
{
}
~WeakToken ()
{
debug_assert(!strong_(delegator));
}
WeakToken &operator =(TokenType &&token_)
{
auto token = std::move(token_);
delegator = weak(token.delegator);
return *this;
}
WeakToken &operator =(WeakToken &&token_)
{
delegator = std::move(token_.delegator);
return *this;
}
void release(Delegate *delegate)
{
if (auto delegator_ = strong_(delegator))
delegator_->eraseDelegate(delegate);
delegator = {};
}
auto &get () { return delegator; }
auto &get () const { return delegator; }
template<typename T>
auto strong ()
{
return cast_to_strong<T>(delegator);
}
template<typename T>
auto strong () const
{
return cast_to_strong<T>(delegator);
}
} ;
template<typename As_, typename Token_, typename Into_>
struct TokenTyped : Token_
{
using Super = Token_;
using As = As_;
using Into = Into_;
using Token = Token_;
TokenTyped (const TokenTyped &) = delete;
TokenTyped() {}
TokenTyped(Token &&token_) :
Super(std::move(token_))
{
}
TokenTyped(TokenTyped &&token_) :
Super(std::move(token_))
{
}
TokenTyped &operator =(Token &&token_)
{
Super::operator=(std::move(token_));
return *this;
}
TokenTyped &operator =(TokenTyped &&token_)
{
Super::operator=(std::move(token_));
return *this;
}
auto strong ()
{
return Super::template strong<As>();
}
auto strong () const
{
return Super::template strong<As>();
}
auto ptr ()
{
return Super::template ptr<As>();
}
auto ptr () const
{
return Super::template ptr<As>();
}
auto operator ->() { return ptr(); }
const auto operator ->() const { return ptr(); }
} ;
template<typename T, typename U, typename V>
auto strong_token(TokenTyped<T, U, V> &t)
{
return t.strong();
}
template<typename Delegate_>
struct Delegator
{
using Delegate = Delegate_;
using Delegates = SafeIteration<List<Delegate *>>;
using TokenType = Token<Delegate>;
Delegates delegates;
virtual TokenType addDelegate(Delegate *delegate) = 0;
virtual void insertDelegate(Delegate *delegate)
{
delegates.push_back(delegate);
}
virtual void eraseDelegate(Delegate *delegate)
{
list_erase_value_one(delegates, delegate);
}
template<typename F>
void eachDelegate(F &&f)
{
for (auto *delegate: delegates)
f(delegate);
}
} ;
template<typename As_, typename Super_>
struct SpecializedDelegator : Super_
{
using Super = Super_;
using DelegatorType = Super;
using Delegate = typename Super::Delegate;
using TokenType = typename Super::TokenType;
using As = As_;
static void eraser(const StrongPtr<DelegatorType> &delegator, Delegate *delegate)
{
delegator->eraseDelegate(delegate);
}
TokenType addDelegate(Delegate *delegate) override
{
this->insertDelegate(delegate);
return TokenType(
std::static_pointer_cast<DelegatorType>(
strong_this(
static_cast<As *>(this)
)
)
);
}
} ;
template<typename Super_>
struct LockingDelegatorOf : Super_
{
TestMutex m;
using Super = Super_;
using Delegate = typename Super::Delegate;
using TokenType = typename Super::TokenType;
void insertDelegate(Delegate *delegate) override
{
auto l = lock_of(m);
return Super::insertDelegate(delegate);
}
void eraseDelegate(Delegate *delegate) override
{
auto l = lock_of(m);
return Super::eraseDelegate(delegate);
}
template<typename F>
void foreach(F &&f)
{
auto l = lock_of(m);
Super::foreach(std::forward<F>(f));
}
} ;
template<typename As, typename Delegate>
using LockingDelegator =
SpecializedDelegator<As, LockingDelegatorOf<Delegator<Delegate>>>;
} // namespace
} // namespace
} // namespace

141
tjp/core/delegate/_tests/Delegate+Single.cpp Normal file
View File

@@ -0,0 +1,141 @@
// TJP COPYRIGHT HEADER
#include <tjp/core/delegate/Delegate+Single.hpp>
#include <tjp/core/testing/catch.hpp>
#include <tjp/core/containers/Function.h>
#include <tjp/core/algorithm/unused.hpp>
#include <tjp/core/log/Log.h>
#include <tjp/core/log/LogOf.h>
#include <iostream>
namespace tjp::core::delegate_single {
namespace T1 {
struct D;
struct A {
A(const StrongPtr<D> &d);
~A();
using Token = Token<A>;
Token token;
int i;
};
struct D :
SpecializedDelegator<D, Delegator<A>>,
StrongThis<D>
{
} ;
A::A(const StrongPtr<D> &d) :
token(d->addDelegate(this))
{
i = 5;
}
A::~A()
{
token.release(this);
}
SCENARIO("delegate_single")
{
// xLogActivateStory("core::delegate");
GIVEN("structs")
{
auto d = strong<D>();
A a(d);
REQUIRE(a.i == 5);
REQUIRE(a.token.delegator == d);
}
GIVEN("sizeof")
{
sLogTest("testing", logVar(sizeof(std::any)) << logVar(sizeof(A)) << logVar(sizeof(D)) << logVar(sizeof(A::Token)));
}
}
} // namespace
namespace T2 {
struct A {
int a;
} ;
struct B : A, StrongThis<B> {
int b;
} ;
void doMemory()
{
std::this_thread::sleep_for(std::chrono::milliseconds(10));
using F = Function<void()>;
auto a = strong<A>();
F f = [a]() {
};
f();
std::this_thread::sleep_for(std::chrono::milliseconds(10));
}
SCENARIO("delegate_single_memory")
{
// xLogActivateStory("core::delegate");
GIVEN("structs")
{
auto b = strong<B>();
b->a = 1;
b->b = 2;
auto a = strong_ptr_cast<A>(b);
REQUIRE(a->a == 1);
auto ab = std::static_pointer_cast<B>(a);
REQUIRE(ab->b == 2);
}
GIVEN("memory")
{
doMemory();
}
GIVEN("sizeof")
{
std::this_thread::sleep_for(std::chrono::milliseconds(250));
using F = Function<void()>;
auto a = strong<A>();
F f = [a]() {};
auto g = []() {};
unused(a);
unused(f);
unused(g);
std::this_thread::sleep_for(std::chrono::milliseconds(250));
sLogTest("testing", logVar(sizeof(std::any)) << logVar(sizeof(Token<A>)) << logVar(sizeof(a)) << logVar(sizeof(F)) << logVar(sizeof(f))<< logVar(sizeof(g)));
}
}
} // namespace
} // namespace

201
tjp/core/delegate/_tests/Delegate.cpp Normal file
View File

@@ -0,0 +1,201 @@
// TJP COPYRIGHT HEADER
#include <tjp/core/delegate/Delegate.hpp>
#include <tjp/core/testing/catch.hpp>
#include <tjp/core/containers/Function.h>
#include <tjp/core/algorithm/unused.hpp>
#include <tjp/core/log/Log.h>
#include <tjp/core/log/LogOf.h>
#include <iostream>
namespace tjp::core::delegate {
namespace T1 {
struct D;
struct To
{
To();
int i;
} ;
struct A : To {
A(const StrongPtr<D> &d);
A(){}
~A();
using Token = Token<To>;
Token token;
};
struct WA : To {
WA(const StrongPtr<D> &d);
WA() {}
~WA();
using Token = WeakToken<Token<To>>;
Token token;
};
struct D :
SpecializedDelegator<D, Delegator<To>>,
StrongThis<D>
{
} ;
To::To()
{
i = 5;
}
A::A(const StrongPtr<D> &d) :
token(d->addDelegate(this))
{
}
A::~A()
{
token.release(this);
}
WA::WA(const StrongPtr<D> &d) :
token(d->addDelegate(this))
{
}
WA::~WA()
{
token.release(this);
}
SCENARIO("delegate")
{
// xLogActivateStory("core::delegate");
GIVEN("strong")
{
auto d = strong<D>();
A a(d);
A b;
REQUIRE(a.i == 5);
REQUIRE(a.token.delegator == d);
{
TokenTyped<D, Token<To>> ad(d->addDelegate(&b));
REQUIRE(strong_token(ad) == d);
ad.release(&b);
}
}
GIVEN("weak")
{
auto d = strong<D>();
WA a(d);
REQUIRE(a.i == 5);
REQUIRE(strong(a.token.get()) == d);
}
GIVEN("weak")
{
auto d = strong<D>();
WA a(d);
REQUIRE(a.i == 5);
REQUIRE(strong(a.token.get()) == d);
}
GIVEN("sizeof")
{
sLogTest("testing", logVar(sizeof(std::any)) << logVar(sizeof(A)) << logVar(sizeof(D)) << logVar(sizeof(A::Token)));
}
}
} // namespace
namespace T2 {
struct A {
int a;
} ;
struct B : A, StrongThis<B> {
int b;
} ;
void doMemory()
{
std::this_thread::sleep_for(std::chrono::milliseconds(10));
using F = Function<void()>;
auto a = strong<A>();
F f = [a]() {
};
f();
std::this_thread::sleep_for(std::chrono::milliseconds(10));
}
SCENARIO("delegate_memory")
{
// xLogActivateStory("core::delegate");
GIVEN("structs")
{
auto b = strong<B>();
b->a = 1;
b->b = 2;
auto a = strong_ptr_cast<A>(b);
REQUIRE(a->a == 1);
auto ab = std::static_pointer_cast<B>(a);
REQUIRE(ab->b == 2);
}
GIVEN("memory")
{
doMemory();
}
GIVEN("sizeof")
{
std::this_thread::sleep_for(std::chrono::milliseconds(250));
using F = Function<void()>;
auto a = strong<A>();
F f = [a]() {};
auto g = []() {};
unused(a);
unused(f);
unused(g);
std::this_thread::sleep_for(std::chrono::milliseconds(250));
sLogTest("testing", logVar(sizeof(std::any)) << logVar(sizeof(Token<A>)) << logVar(sizeof(a)) << logVar(sizeof(F)) << logVar(sizeof(f))<< logVar(sizeof(g)));
}
}
} // namespace
} // namespace

16
tjp/core/endian/Endian.cpp Executable file
View File

@@ -0,0 +1,16 @@
// TJP COPYRIGHT HEADER
#include "Endian.h"
using namespace tjp::core;
void endian::swap (char *buffer, int size)
{
int i, j;
for (i = 0, j = size-1 ; i < size/2; ++i, --j)
{
char switcheroo = buffer[i];
buffer[i] = buffer[j];
buffer[j] = switcheroo;
}
}

72
tjp/core/endian/Endian.h Executable file
View File

@@ -0,0 +1,72 @@
// TJP COPYRIGHT HEADER
#ifndef __Utilities_Endian_h__
#define __Utilities_Endian_h__
namespace tjp {
namespace core {
namespace endian {
/**
* Swaps a buffer that represents one variable
*/
void swap (char *, int size);
/**
* Swaps the specified buffer if the processor is big endian.
*/
inline void toLittleEndian (char *buffer, int size)
{
#ifdef __BIG_ENDIAN__
swap(buffer, size);
#endif
}
template<typename T>
inline void toLittleEndian (T &t)
{
#ifdef __BIG_ENDIAN__
swap((char*)&t, sizeof(T));
#endif
return t;
}
/**
* Swaps the specified buffer if the processor is little endian.
*/
inline void toBigEndian (char *buffer, int size)
{
#ifndef __BIG_ENDIAN__
swap(buffer, size);
#endif
}
template<typename T>
inline void toBigEndian (T &t)
{
#ifndef __BIG_ENDIAN__
swap((char*)&t, sizeof(T));
#endif
return t;
}
/**
* Swaps a single variable.
*/
template<typename T>
inline T &swap (T &t)
{
#ifdef __BIG_ENDIAN__
swap((char*)&t, sizeof(T));
#endif
return t;
}
} // namespace endian
} // namespace core
} // namespace
#endif

75
tjp/core/events/Signal.cpp Normal file
View File

@@ -0,0 +1,75 @@
// TJP COPYRIGHT HEADER
#include "Signal.hpp"
#include <tjp/core/algorithm/set_erase.hpp>
#include <tjp/core/threads/TestMutex.hpp>
namespace tjp::core::signal {
Listener *Signal::insert(Callback &&callback)
{
auto lock = lock_of(mutex);
auto listener = strong<Listener>(std::move(callback));
auto r = listeners.emplace(listener);
debug_assert(r.second);
return ptr_of(listener);
}
void Signal::erase(Listener *listener)
{
auto lock = lock_of(mutex);
invalidated = true;
set_erase(listeners, listener);
}
void Signal::signal()
{
// this is correct, below, we reset to begin if it is the special case of the end
Listeners::iterator i = listeners.end();
invalidated = false;
StrongPtr<Listener> listener = nullptr;
do
{
mutex.lock();
if (i == listeners.end())
{
i = listeners.begin();
}
else
{
if (invalidated)
{
invalidated = false;
i = listeners.upper_bound(listener);
}
else
{
++i;
}
}
if (i == listeners.end())
break;
listener = *i;
mutex.unlock();
(*listener)();
}
while (true);
mutex.unlock();
}
void Signal::clear()
{
mutex.lock();
listeners.clear();
}
} // namespace

21
tjp/core/events/Signal.h Normal file
View File

@@ -0,0 +1,21 @@
// TJP COPYRIGHT HEADER
#pragma once
#include <tjp/core/containers/Function.h>
namespace tjp::core {
namespace signal {
using Callback = Function<void()>;
using Listener = Callback;
struct Signal;
} // namespace
using signal::Signal;
using SignalListener = signal::Listener;
} // namespace

40
tjp/core/events/Signal.hpp Normal file
View File

@@ -0,0 +1,40 @@
// TJP COPYRIGHT HEADER
#pragma once
#include "Signal.h"
#include <tjp/core/containers/Set.hpp>
#include <tjp/core/threads/TestMutex.hpp>
#include <tjp/core/containers/ByPtr.hpp>
namespace tjp::core {
namespace signal {
using Callback = Function<void()>;
using Listener = Callback;
struct Signal
{
[[no_unique_address]]
TestMutex mutex;
typedef SetSortBy<StrongPtr<Listener>, core::ByPtr<Listener>> Listeners;
Listeners listeners;
bool invalidated = false;
Listener *insert(Callback &&callback);
void erase(Listener *listener);
// todo change to emit
void signal();
void clear();
} ;
} // namespace
using signal::Signal;
using SignalListener = signal::Listener;
} // namespace

188
tjp/core/extensible/Extensible+CacheViaArray.h Normal file
View File

@@ -0,0 +1,188 @@
// TJP COPYRIGHT HEADER
#pragma once
#include "Extensible+Enumerated.h"
#ifdef _DEBUG
void onExtensibleCacheNew_(void *);
void onExtensibleCache_(void *, size_t);
void onExtensibleCacheDelete_(void *, size_t);
void onExtensibleCacheClear_(void *, size_t);
inline void onExtensibleCacheNew(void *t) { onExtensibleCacheNew_(t); }
inline void onExtensibleCache(void *t, size_t v) { onExtensibleCache_(t, v); }
inline void onExtensibleCacheDelete(void *t, size_t v) { onExtensibleCacheDelete_(t, v); }
inline void onExtensibleCacheClear(void *t, size_t v) { onExtensibleCacheDelete_(t, v); }
#else
inline void onExtensibleCacheNew(void *) {}
inline void onExtensibleCache(void *, size_t) {}
inline void onExtensibleCacheDelete(void *, size_t) {}
inline void onExtensibleCacheClear(void *, size_t) {}
#endif
struct ExtensibleCacheViaArray
{
[[no_unique_address]]
TestMutex mutex;
typedef ExtensionInterface *ExtensionInterfacePtr;
constexpr static size_t REDIRECT_TYPE_BITSIZE = 8;
constexpr static size_t REDIRECTS_PER_BYTE = 8 / REDIRECT_TYPE_BITSIZE;
// leave one left over for the size variable
constexpr static size_t MAX_TYPE = 48 - REDIRECTS_PER_BYTE;
constexpr static size_t REDIRECT_CACHE_BYTE_SIZE = MAX_TYPE / REDIRECTS_PER_BYTE;
constexpr static size_t
MAX_CACHE = (1 << REDIRECT_TYPE_BITSIZE) - 1
- 1; // 0xF is NULL_REDIRECT
constexpr static u8 CACHE_MASK = (1 << REDIRECT_TYPE_BITSIZE) - 1;
constexpr static size_t NULL_REDIRECT = (1 << REDIRECT_TYPE_BITSIZE) - 1;
// using Cache = InPlaceArray<ExtensionInterfacePtr, MAX_CACHE>;
using Cache = StackArray<ExtensionInterfacePtr, 2>;
// using Cache = Vector<ExtensionInterfacePtr>;
Cache cache;
using Redirects = u8[REDIRECT_CACHE_BYTE_SIZE];
Redirects redirect;
ExtensibleCacheViaArray()
{
onExtensibleCacheNew(this);
memset(redirect, CACHE_MASK, REDIRECT_CACHE_BYTE_SIZE);
}
~ExtensibleCacheViaArray()
{
debug_assert(cache.empty());
}
void clear()
{
onExtensibleCacheDelete(this, cache.size());
memset(redirect, 0xFF, REDIRECT_CACHE_BYTE_SIZE);
for (auto p : cache)
delete p;
cache.clear();
}
inline
size_t getRedirect(size_t index)
{
debug_assert(index < MAX_TYPE);
if constexpr(REDIRECTS_PER_BYTE == 1)
{
return redirect[index] ;
}
else
{
auto redirectByte = index / REDIRECTS_PER_BYTE;
auto redirectNibble = index % REDIRECTS_PER_BYTE;
auto redirectShift = redirectNibble * REDIRECT_TYPE_BITSIZE;
auto byte = redirect[redirectByte] >> redirectShift;
return byte & CACHE_MASK;
}
}
inline
void setRedirect(size_t index, u8 value)
{
debug_assert(index < MAX_TYPE);
if constexpr(REDIRECTS_PER_BYTE == 1)
{
redirect[index] = value;
}
else
{
auto redirectByte = index / REDIRECTS_PER_BYTE;
auto redirectNibble = index % REDIRECTS_PER_BYTE;
auto redirectShift = redirectNibble * REDIRECT_TYPE_BITSIZE;
u8 writeBits = 0xF << redirectShift;
u8 keepBits = ~writeBits;
auto &byte = redirect[redirectByte];
byte = (byte & keepBits) | (value << redirectShift);
}
}
template<typename T>
T *find()
{
auto lock = lock_of(mutex);
(void)lock;
auto index = T::getExtensionIndex();
debug_assert(index < MAX_TYPE);
auto redirect = getRedirect(index);
if (redirect == NULL_REDIRECT)
return nullptr;
debug_assert(redirect < MAX_CACHE);
return static_cast<T *>(cache[redirect]);
}
template<typename T>
void insert(T *t)
{
auto lock = lock_of(mutex);
(void)lock;
auto index = T::getExtensionIndex();
debug_assert(index < MAX_TYPE);
auto redirect = getRedirect(index);
if (redirect != NULL_REDIRECT)
{
debug_assert(cache[redirect] == nullptr);
cache[redirect] = t;
return;
}
redirect = cache.size();
cache.push_back(t);
onExtensibleCache(this, cache.size());
debug_assert(redirect < MAX_CACHE);
setRedirect(index, redirect);
debug_assert(redirect < MAX_CACHE);
}
template<typename T>
void erase(T *t)
{
auto lock = lock_of(mutex);
(void)lock;
auto index = T::getExtensionIndex();
debug_assert(index < MAX_TYPE);
auto redirect = getRedirect(index);
if (redirect == NULL_REDIRECT)
return ;
debug_assert(redirect < MAX_CACHE);
cache[redirect] = nullptr;
}
} ;
using ExtensibleCache = ExtensibleCacheViaArray;
template<typename T>
using Extension = ExtensionEnumerated<T>;

72
tjp/core/extensible/Extensible+CacheViaArrayAndMap.h Normal file
View File

@@ -0,0 +1,72 @@
// TJP COPYRIGHT HEADER
#pragma once
#include "Extensible+CacheViaMap.h"
#include "Extensible+Enumerated.h"
struct ExtensibleCacheViaArrayAndMap
{
typedef ExtensionInterface *ExtensionInterfacePtr;
constexpr static size_t MAX_CACHE=64;
ExtensionInterfacePtr cache[MAX_CACHE];
ExtensibleCacheViaMap map;
ExtensibleCacheViaArrayAndMap()
{
memset(cache, 0, sizeof(ExtensionInterfacePtr) * MAX_CACHE);
}
~ExtensibleCacheViaArrayAndMap()
{
clear();
}
void clear()
{
auto *begin = &cache[0];
auto *end = begin + MAX_CACHE;
for (auto *p=begin; p!=end; ++p)
{
if (*p)
{
delete *p;
*p = nullptr;
}
}
map.clear();
}
template<typename T>
T *find()
{
auto index = T::getExtensionIndex();
if (index < MAX_CACHE)
return static_cast<T *>(cache[index]);
return map.find<T>();
}
template<typename T>
void insert(T *t)
{
auto index = T::getExtensionIndex();
if (index < MAX_CACHE)
{
cache[index] = t;
}
else
{
debug_assert(false);
map.insert(t);
}
}
} ;
using ExtensibleCache = ExtensibleCacheViaArrayAndMap;
template<typename T>
using Extension = ExtensionEnumerated<T>;

51
tjp/core/extensible/Extensible+CacheViaMap.h Normal file
View File

@@ -0,0 +1,51 @@
// TJP COPYRIGHT HEADER
#pragma once
#include "Extensible+TypeInfo.h"
struct ExtensibleCacheViaMap
{
typedef ExtensionInterface *ExtensionInterfacePtr;
std::map<std::type_index, ExtensionInterfacePtr> extensions;
~ExtensibleCacheViaMap()
{
clear();
}
void clear()
{
for (auto &kv : extensions)
delete kv.second;
extensions.clear();
}
template<typename T>
T *find()
{
const auto key = type_id<T>();
auto i = extensions.find(key);
if (i == extensions.end())
return nullptr;
return static_cast<T *>(i->second);
}
template<typename T>
void insert(T *t)
{
const auto key = type_id<T>();
debug_assert(extensions.find(key) == extensions.end());
extensions.emplace(key, t);
}
} ;
//using ExtensibleCache = ExtensibleCacheViaMap;
//
//template<typename T>
//using Extension = ExtensionTypeInfo<T>;

27
tjp/core/extensible/Extensible+Enumerated.h Normal file
View File

@@ -0,0 +1,27 @@
// TJP COPYRIGHT HEADER
#pragma once
template<typename T>
struct ExtensionEnumerated : ExtensionInterface
{
static size_t extension_index;
static size_t getExtensionIndex();
} ;
size_t getNextExtensionIndex();
template<typename T>
size_t ExtensionEnumerated<T>::extension_index = -1;
template<typename T>
size_t ExtensionEnumerated<T>::getExtensionIndex()
{
if (extension_index != -1)
return extension_index;
extension_index = getNextExtensionIndex();
return extension_index;
}

7
tjp/core/extensible/Extensible+TypeInfo.h Normal file
View File

@@ -0,0 +1,7 @@
// TJP COPYRIGHT HEADER
template<typename T>
struct ExtensionTypeInfo : ExtensionInterface
{
} ;

67
tjp/core/extensible/Extensible.cpp Normal file
View File

@@ -0,0 +1,67 @@
// TJP COPYRIGHT HEADER
#include "Extensible.h"
#include <atomic>
#include <tjp/core/log/Log.h>
#include <tjp/core/log/LogOf.h>
namespace tjp::core {
std::atomic<size_t> nextExtensionIndex = 0;
size_t getNextExtensionIndex()
{
sLogDebug("core::extensible", logVar(nextExtensionIndex));
return nextExtensionIndex++;
}
constexpr size_t MAX_CACHE_STATISTICS = 64;
static std::array<size_t, MAX_CACHE_STATISTICS+1> numPerSizeCurrent = {0};
static std::array<size_t, MAX_CACHE_STATISTICS+1> numPerSizeTotals = {0};
void onExtensibleCache_(void *t, size_t size)
{
if (size < MAX_CACHE_STATISTICS)
{
if (size > 0)
{
numPerSizeCurrent[size-1]--;
numPerSizeTotals[size-1]--;
}
numPerSizeCurrent[size]++;
numPerSizeTotals[size]++;
}
sLogDebug("core::extensible", "C " << logVar(numPerSizeCurrent));
sLogDebug("core::extensible", "T " << logVar(numPerSizeTotals));
}
void onExtensibleCacheNew_(void *t)
{
onExtensibleCache_(t, 0);
}
void onExtensibleCacheClear_(void *t, size_t size)
{
if (size < MAX_CACHE_STATISTICS)
{
numPerSizeCurrent[size]--;
numPerSizeTotals[size]--;
numPerSizeCurrent[0]++;
numPerSizeTotals[0]++;
}
}
void onExtensibleCacheDelete_(void *t, size_t size)
{
numPerSizeCurrent[size]--;
sLogDebug("core::extensible", "C " << logVar(numPerSizeCurrent));
sLogDebug("core::extensible", "T " << logVar(numPerSizeTotals));
}
} // namespace

269
tjp/core/extensible/Extensible.h Normal file
View File

@@ -0,0 +1,269 @@
// TJP COPYRIGHT HEADER
#pragma once
#include <map>
#include <string.h>
#include <tjp/core/exception/Exception.hpp>
#include <tjp/core/assert/debug_assert.h>
#include <tjp/core/algorithm/remove_const_of_var.hpp>
#include <tjp/core/types/Types.h>
#include <tjp/core/debug/Debug.h>
#include <tjp/core/containers/InPlaceArray.hpp>
#include <tjp/core/containers/StackArray.hpp>
#include <tjp/core/threads/SpinMutex.hpp>
#include <tjp/core/threads/Lock.hpp>
#include <tjp/core/threads/TestMutex.hpp>
#include "ExtensibleWith.h"
namespace tjp::core {
struct ExtensionInterface
{
ExtensionInterface (const ExtensionInterface &) = delete;
ExtensionInterface () {}
virtual ~ExtensionInterface () {}
virtual void initializeExtension() {}
} ;
template<typename T>
void initializeExtension(T *t)
{
t->initializeExtension();
}
//#include "Extensible+CacheViaArrayAndMap.h"
#include "Extensible+CacheViaArray.h"
// -----------------
struct ExtensibleBaseDefault
{
virtual ~ExtensibleBaseDefault () {};
} ;
template<typename B=ExtensibleBaseDefault>
struct Extensor : B
{
typedef B Super;
mutable ExtensibleCache extensions;
Extensor () { }
template<typename ...T>
Extensor(T &&...t) :
B(std::forward<T>(t)...) {}
virtual ~Extensor ()
{
extensions.clear();
}
template<typename U, typename T, typename std::enable_if<std::is_constructible<T, U*>::value>::type* = nullptr>
T &getExtensionTU (U *self) const
{
if (auto extension = extensions.find<T>())
return *extension;
T *t = new T(
static_cast<U *>(self)
);
extensions.insert(t);
initializeExtension(t);
return *t;
}
template<typename U, typename T, typename std::enable_if<!std::is_constructible<T, U*>::value>::type* = nullptr>
T &getExtensionTU (U *self) const
{
if (auto extension = extensions.find<T>())
return *extension;
throw tjp::core::Exception("Extension not available");
}
template<typename U, typename T, typename std::enable_if<std::is_constructible<T, U*>::value>::type* = nullptr>
T &getExtensionU () const
{
if (auto extension = extensions.find<T>())
return *extension;
T *t = new T(
static_cast<U *>(remove_const_of_var(this))
);
extensions.insert(t);
initializeExtension(t);
return *t;
}
template<typename U, typename T, typename std::enable_if<!std::is_constructible<T, U*>::value>::type* = nullptr>
T &getExtensionU () const
{
if (auto extension = extensions.find<T>())
return *extension;
throw tjp::core::Exception("Extension not available");
}
template<typename U, typename T, typename std::enable_if<std::is_constructible<T, U*>::value>::type* = nullptr>
bool hasExtensionU () const
{
return true;
}
template<typename U, typename T, typename std::enable_if<!std::is_constructible<T, U*>::value>::type* = nullptr>
bool hasExtensionU () const
{
return extensions.find<T>();
}
template<typename U, typename T, typename TI, typename ...V>
T &makeExtensionTU (const U *self, V &&...v) const
{
if (auto extension = extensions.find<T>())
return *extension;
T *t = new TI(
static_cast<U *>(remove_const_of_var(self)),
v...
);
extensions.insert(t);
initializeExtension(t);
return *t;
}
template<typename U, typename T, typename TI, typename ...V>
T &makeExtensionU (V &&...v) const
{
if (auto extension = extensions.find<T>())
return *extension;
T *t = new TI(
static_cast<U *>(remove_const_of_var(this)),
v...
);
extensions.insert(t);
initializeExtension(t);
return *t;
}
template<typename U, typename T>
T withExtensionU () const
{
return T(
ExtendingWith {},
static_cast<U *>(remove_const_of_var(this))
);
}
template<typename T>
void eraseExtension ()
{
if (auto extension = extensions.find<T>())
{
extensions.erase(extension);
delete extension;
}
}
};
template<typename P, typename B=ExtensibleBaseDefault>
struct Extensible : Extensor<B>
{
typedef Extensor<B> Extensor_;
typedef Extensible<P,B> Extensible_;
Extensible () {}
template<typename ...T>
Extensible(T &&...t) :
Extensor_(std::forward<T>(t)...) {}
template<typename T>
bool hasExtension () const
{
return this->template hasExtensionU<P, T>();
}
template<typename T>
T &getExtension () const
{
return this->template getExtensionU<P, T>();
}
template<typename T, typename TI, typename ...V>
T &makeExtension (V &&...v) const
{
return this->template makeExtensionU<P, T, TI>(v...);
}
template<typename T, typename ...V>
T &makeExtension (V &&...v) const
{
return this->template makeExtensionU<P, T, T>(v...);
}
template<typename T>
T withExtension () const
{
return this->template withExtensionU<P, T>();
}
} ;
template<typename B, typename U>
struct Extensibled : B
{
typedef B Super;
typedef Extensibled<B, U> Extensible_;
template<typename ...T>
Extensibled(T &&...t) :
B(std::forward<T>(t)...) {}
Extensibled() {}
template<typename T>
T &getExtension () const
{
return this->template getExtensionU<U, T>();
}
template<typename T>
bool hasExtension () const
{
return this->template hasExtensionU<U, T>();
}
template<typename T, typename TI, typename ... V>
T &makeExtension (V &&...v) const
{
return this->template makeExtensionU<U, T, TI>(v...);
}
template<typename T, typename ... V>
T &makeExtension (V &&...v) const
{
return this->template makeExtensionU<U, T, T>(v...);
}
template<typename T>
T withExtension () const
{
return this->template withExtensionU<U, T>();
}
} ;
} // namespace

33
tjp/core/extensible/ExtensibleWith.h Normal file
View File

@@ -0,0 +1,33 @@
// TJP COPYRIGHT HEADER
#pragma once
#include <tjp/core/algorithm/remove_const_of_var.hpp>
namespace tjp::core {
template<typename T>
struct WithExtension {};
struct ExtendingWith {};
template<typename P>
struct ExtensibleWith
{
template<typename U, typename T>
T withExtensionU () const
{
return T(
ExtendingWith {},
static_cast<U *>(remove_const_of_var(this))
);
}
template<typename T>
T withExtension () const
{
return withExtensionU<P, T>();
}
} ;
} // namespace

130
tjp/core/extensible/_tests/Extensible.cpp Normal file
View File

@@ -0,0 +1,130 @@
// TJP COPYRIGHT HEADER
#include <tjp/core/extensible/Extensible.h>
#include <tjp/core/testing/catch.hpp>
#include <tjp/core/log/Log.h>
#include <tjp/core/log/LogOf.h>
namespace tjp::core {
namespace {
struct A : Extensible<A>
{
int num = 0;
} ;
struct B : Extensibled<A, B>
{
} ;
struct C : Extensibled<B, C>
{
} ;
struct A_Ext : Extension<A_Ext>
{
A_Ext (A *a) { a->num++; }
char val() const { return 'A'; }
} ;
struct B_Ext : Extension<B_Ext>
{
B_Ext (B *b) { b->num++; }
char val() const { return 'B'; }
} ;
struct C_Ext : Extension<C_Ext>
{
C_Ext (C *c) { c->num++; }
char val() const { return 'C'; }
} ;
SCENARIO("extensible")
{
xLogActivateStory("core::extensible");
GIVEN("sizeof")
{
sLogTest("testing", logVar(sizeof(Extensible<A>)) << logVar(sizeof(ExtensibleCache)));
sLogTest("testing", logVar(sizeof(ExtensibleCache::Cache)) << logVar(sizeof(ExtensibleCache::Redirects)));
}
GIVEN("a get extension")
{
A a;
B b;
C c;
auto val = a.getExtension<A_Ext>().val() ;
REQUIRE(val == 'A');
REQUIRE(A_Ext::extension_index == 0);
REQUIRE(b.getExtension<B_Ext>().val() == 'B');
REQUIRE(B_Ext::extension_index == 1);
// REQUIRE(c.getExtension<C_Ext>().val() == 'C');
val = c.getExtension<C_Ext>().val();
REQUIRE(val == 'C');
REQUIRE(C_Ext::extension_index == 2);
REQUIRE(a.num == 1);
REQUIRE(b.num == 1);
REQUIRE(c.num == 1);
val = c.getExtension<A_Ext>().val() ;
REQUIRE(val == 'A');
val = c.getExtension<B_Ext>().val() ;
REQUIRE(val == 'B');
val = c.getExtension<C_Ext>().val() ;
REQUIRE(val == 'C');
val = c.getExtension<A_Ext>().val() ;
REQUIRE(val == 'A');
val = c.getExtension<B_Ext>().val() ;
REQUIRE(val == 'B');
val = c.getExtension<C_Ext>().val() ;
REQUIRE(val == 'C');
REQUIRE(c.num == 3);
}
GIVEN("a make extension")
{
A a;
B b;
C c;
auto val = a.makeExtension<A_Ext>().val() ;
REQUIRE(val == 'A');
REQUIRE(b.makeExtension<B_Ext>().val() == 'B');
REQUIRE(c.makeExtension<C_Ext>().val() == 'C');
val = a.getExtension<A_Ext>().val() ;
REQUIRE(val == 'A');
REQUIRE(a.num == 1);
REQUIRE(b.num == 1);
REQUIRE(c.num == 1);
}
}
} // namespace
} // namespace

98
tjp/core/file/Contents.h Normal file
View File

@@ -0,0 +1,98 @@
// TJP COPYRIGHT HEADER
#pragma once
#include <tjp/core/containers/Vector.h>
#include <tjp/core/string/StringView.h>
#include <tjp/core/string/String.h>
#include <tjp/core/const_expr/Str.hpp>
namespace tjp::core::file {
using Contents = Vector<char>;
struct ContentsView
{
using value_type = char;
using pointer_type = const value_type *;
const char *data_;
const size_t size_;
constexpr
pointer_type data() const
{
return data_;
}
constexpr
size_t size() const
{
return size_;
}
constexpr ContentsView() :
data_((char *)nullptr),
size_(0)
{
}
template<typename T>
constexpr ContentsView(const T *data, const size_t size) :
data_((pointer_type)data),
size_(size * sizeof(T))
{
}
template<typename T>
ContentsView(const Vector<T> &v) :
ContentsView(v.data(), v.size())
{
}
template<typename T>
constexpr ContentsView(const T &v) :
ContentsView(v.data(), v.size())
{
}
constexpr ContentsView(const char *data) :
ContentsView(data, core::const_expr::length(data))
{
}
constexpr ContentsView(const ContentsView &v) :
ContentsView(v.data(), v.size())
{
}
constexpr ContentsView(const StringView &v) :
ContentsView(v.data(), v.size())
{
}
ContentsView(const String &v) :
ContentsView(v.c_str(), v.size())
{
}
operator Contents() const
{
return Contents(data_, data_ + size_);
}
operator StringView() const
{
return StringView(data_, size_);
}
operator String() const
{
return String(data_, size_);
}
} ;
} // namespace

24
tjp/core/file/Directories.h Normal file
View File

@@ -0,0 +1,24 @@
// TJP COPYRIGHT HEADER
#pragma once
#include <tjp/core/system/System.h>
#if defined(SYS_APPLE)
#include "apple/Directories.h"
namespace tjp::core::file {
using namespace file::apple;
}
#else
#include "linux/Directories.h"
namespace tjp::core::file {
using namespace file::linux;
}
#endif

158
tjp/core/file/File.cpp Normal file
View File

@@ -0,0 +1,158 @@
// TJP COPYRIGHT HEADER
#include "File.h"
#include <iostream>
#include <fstream>
#include <sys/stat.h>
#include <sys/types.h>
#include <utime.h>
#include <tjp/core/exception/Exception.hpp>
#include <tjp/core/string/to_string.hpp>
#include <tjp/core/string/StringView+concat.hpp>
namespace tjp::core::file {
TJP_CORE_HEADER_ONLY_INLINE
Optional<size_t> size(const StringView& path)
{
std::ifstream file(String(path), std::ios::ate | std::ios::binary);
bool exists = (bool)file;
if (!exists || !file.is_open())
return {};
size_t fileSize = (size_t)file.tellg();
return fileSize;
}
TJP_CORE_HEADER_ONLY_INLINE
Optional<Contents> readRange(const StringView& path, s64 offset, size_t length)
{
std::ifstream file(String(path), std::ios::ate | std::ios::binary);
bool exists = (bool)file;
if (!exists || !file.is_open())
return {};
size_t fileSize = (size_t)file.tellg();
if (offset < 0)
offset += s64(fileSize);
if (offset < 0)
return {};
if (fileSize < offset + length)
return {};
Contents buffer(length);
file.seekg(offset);
file.read(buffer.data(), buffer.size());
file.close();
return buffer;
};
TJP_CORE_HEADER_ONLY_INLINE
bool pathExists (const StringView &file)
{
struct stat status;
return (stat (file.data(), &status)==0);
}
TJP_CORE_HEADER_ONLY_INLINE
bool ensurePath (const StringView &base_, const StringView &path)
{
char delimiter = '/';
// find the end of the last portion of the path we need to create
auto finalPos = path.rfind(delimiter);
if (finalPos == path.npos)
return true;
auto base = String(base_);
if (!base.empty() && base.back() != delimiter)
base += delimiter;
decltype(finalPos) pos = 0;
auto colonpos = path.find(':');
auto firstdelimiterpos = path.find(delimiter);
if (colonpos > 0 && firstdelimiterpos > 0 && colonpos < firstdelimiterpos)
{
// Skip the first one if there's a colon ('c:/blah/whaterver')
pos = path.find (delimiter, pos+1);
}
while (pos<finalPos)
{
// find the next part of the path to create
pos = path.find (delimiter, pos+1);
// does it already exist
#ifdef WIN32
// pull it out as a separate string
std::wstring directory_ = toLongPath(path.substr (0, pos));
std::wstring directory = base + directory_;
WIN32_FILE_ATTRIBUTE_DATA data;
bool exists = GetFileAttributesExW (base + directory.c_str(), GetFileExInfoStandard, &data) > 0;
if (exists)
{
// if it does exist, but is not a directory, ensurePath fails
if (!(data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY))
return false;
}
else
{
// if it does not exist create it, if this fails ensurePath fails
if (_wmkdir (directory.c_str())!=0)
return false;
}
#else
auto directory_ = path.substr (0, pos);
auto directory = base + directory_;
struct stat status;
int exists = (stat (directory.c_str(), &status)==0);
if (exists)
{
// if it does exist, but is not a directory, ensurePath fails
if (!(status.st_mode & S_IFDIR))
return false;
}
else
{
// if it does not exist create it, if this fails ensurePath fails
auto result = mkdir (directory.c_str(), (S_IRWXU | S_IRWXG | S_IRWXO));
if (result)
return false;
}
#endif
}
return true;
}
TJP_CORE_HEADER_ONLY_INLINE
bool ensurePath (const StringView &path)
{
return ensurePath("", path);
}
} // namespace

46
tjp/core/file/File.h Normal file
View File

@@ -0,0 +1,46 @@
// TJP COPYRIGHT HEADER
#pragma once
#include <tjp/core/header_only/compile.h>
#include "Contents.h"
#include <tjp/core/string/StringView.h>
#include <tjp/core/string/String.h>
#include <tjp/core/containers/Vector.h>
#include <tjp/core/containers/Optional.h>
#include <tjp/core/types/Types.h>
#include <time.h>
#include "file_exists.hpp"
#include "read.hpp"
#include "write.hpp"
#include "times.hpp"
namespace tjp::core::file {
Optional<size_t> size(const StringView &path);
Optional<Contents> readRange(const StringView &path, s64 offset, size_t length);
Contents readFile(const StringView &path);
bool writeFile(const StringView &path, const ContentsView &contents, const WriteOptions &options = {});
bool pathExists (const StringView &path);
bool ensurePath (const StringView &path);
bool ensurePath (const StringView &base, const StringView &path);
bool deleteFile(const StringView &path);
Vector<String> getFiles(const StringView &path, const Optional<String> &extension);
} // namespace
#ifdef TJP_CORE_HEADER_ONLY
#include "File.cpp"
#endif

328
tjp/core/file/File_Messy.cpp Normal file
View File

@@ -0,0 +1,328 @@
// TJP COPYRIGHT HEADER
#include "File.h"
#include <iostream>
#include <fstream>
#include <sys/stat.h>
#include <sys/types.h>
#include <tjp/core/exception/Exception.hpp>
#include <tjp/core/system/System.h>
#include <tjp/core/string/to_string.hpp>
#include "../random/Random.h"
#include <tjp/core/log/Log.h>
#include <tjp/core/log/LogOf.h>
#include <dirent.h>
#include <sys/types.h>
#include <string>
#include <queue>
#include <filesystem>
#include <utime.h>
namespace tjp::core::file {
std::vector<char> readFile(const StringView& path)
{
std::ifstream file(String(path), std::ios::ate | std::ios::binary);
bool exists = (bool)file;
if (!exists || !file.is_open()) {
throw Exception { "failed to open file " + String(path) };
}
size_t fileSize = (size_t)file.tellg();
std::vector<char> buffer(fileSize);
file.seekg(0);
file.read(buffer.data(), fileSize);
file.close();
return buffer;
};
bool writeFile(const StringView &path_, const ContentsView &contents, const WriteOptions &options)
{
auto path = String(path_);
auto extension = "." + core::to_string(core::StandardRandom.next<u32>());
auto temporary = path + extension;
std::ofstream file(temporary, std::ios::binary);
if (!file)
return false;
file.write(contents.data(), contents.size());
file.close();
remove(path.c_str());
if (rename(temporary.c_str(), path.c_str()))
return false;
if (options.lastModifiedTime)
{
setFileLastModifiedTime(path, options.lastModifiedTime);
}
return true;
}
/*
bool ensurePath (const StringView &path)
{
return ensurePath("", path);
}
bool ensurePath (const StringView &base_, const StringView &path_)
{
sLogDebug("core::file", logVar(base_) << logVar(path_));
char delimiter = '/';
auto path = String(path_);
auto base = String(base_);
if (!base.empty() && base.back() != delimiter)
base += delimiter;
// find the end of the last portion of the path we need to create
auto finalPos = path.rfind (delimiter);
decltype(finalPos) pos = 0;
auto colonpos = path.find(':');
auto firstdelimiterpos = path.find(delimiter);
if (colonpos > 0 && firstdelimiterpos > 0 && colonpos < firstdelimiterpos)
{
// Skip the first one if there's a colon ('c:/blah/whaterver')
pos = path.find (delimiter, pos+1);
}
while (pos<finalPos)
{
// find the next part of the path to create
pos = path.find (delimiter, pos+1);
// does it already exist
#ifdef WIN32
// pull it out as a separate string
std::wstring directory_ = toLongPath(path.substr (0, pos));
std::wstring directory = base + directory_;
WIN32_FILE_ATTRIBUTE_DATA data;
bool exists = GetFileAttributesExW (base + directory.c_str(), GetFileExInfoStandard, &data) > 0;
if (exists)
{
// if it does exist, but is not a directory, ensurePath fails
if (!(data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY))
return false;
}
else
{
// if it does not exist create it, if this fails ensurePath fails
if (_wmkdir (directory.c_str())!=0)
return false;
}
#else
std::string directory_ = path.substr (0, pos);
std::string directory = String(base) + directory_;
sLogDebug("core::file", logVar(path) << logVar(directory));
struct stat status;
int exists = (stat (directory.c_str(), &status)==0);
if (exists)
{
// if it does exist, but is not a directory, ensurePath fails
if (!(status.st_mode & S_IFDIR))
return false;
}
else
{
// if it does not exist create it, if this fails ensurePath fails
auto result = mkdir (directory.c_str(), (S_IRWXU | S_IRWXG | S_IRWXO));
if (result)
return false;
}
#endif
}
return true;
}
bool fileExists (const StringView &file)
{
#ifdef WIN32
WIN32_FILE_ATTRIBUTE_DATA data;
bool exists = GetFileAttributesExW (toLongPath(file).c_str(), GetFileExInfoStandard, &data) > 0;
return exists && !(data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY);
#else
struct stat status;
if (stat (file.data(), &status)==0)
{
// if it is not a directory return true
if (!(status.st_mode & S_IFDIR))
return true;
}
return false;
#endif
}
bool pathExists (const StringView &file)
{
#ifdef WIN32
WIN32_FILE_ATTRIBUTE_DATA data;
bool exists = GetFileAttributesExW (toLongPath(file).c_str(), GetFileExInfoStandard, &data) > 0;
return exists;
#else
struct stat status;
return (stat (file.data(), &status)==0);
#endif
}
*/
bool deleteFile(const StringView &file)
{
return remove(file.data()) == 0;
}
#ifndef SYS_IOS
Vector<String> getFiles(const StringView &path, const Optional<String> &extension)
{
Vector<String> files;
try
{
std::queue<std::filesystem::path> dirs;
std::filesystem::path basePath = std::filesystem::canonical(path);
dirs.push(basePath);
while (!dirs.empty())
{
auto current = dirs.front();
dirs.pop();
for (const auto &entry : std::filesystem::directory_iterator(current))
{
if (entry.is_directory())
{
dirs.push(entry.path());
}
else
if (entry.is_regular_file())
{
if (!extension || entry.path().extension() == *extension) {
files.push_back(std::filesystem::relative(entry.path(), basePath).string());
}
}
}
}
}
catch (std::filesystem::filesystem_error &)
{
std::cerr << "caught std::filesystem::filesystem_error " << std::endl;
}
catch (std::exception &)
{
std::cerr << "caught std::exception " << std::endl;
}
catch (...)
{
std::cerr << "caught unknown exception " << std::endl;
}
return files;
}
#else
Vector<String> getFiles(const StringView &path_, const Optional<String> &extension)
{
auto path = String(path_);
Vector<String> files;
std::queue<std::string> dirs;
dirs.push("");
while (!dirs.empty())
{
auto relativeDir = dirs.front();
std::string absoluteDir = path + relativeDir;
dirs.pop();
DIR *dir = opendir(absoluteDir.c_str());
if (!dir) {
continue;
}
struct dirent *entry;
while ((entry = readdir(dir)) != nullptr)
{
std::string name(entry->d_name);
// skip "." and ".."
if (name == "." || name == "..") {
continue;
}
std::string relativePath =
relativeDir.empty() ?
name :
relativeDir + "/" + name;
if (entry->d_type == DT_DIR)
{
// enqueue subdirectory
dirs.push(relativePath);
}
else
{
if (extension.has_value())
{
if (name.size() >= extension->size() &&
name.compare(name.size() - extension->size(), extension->size(), *extension) == 0)
{
files.emplace_back(relativePath.c_str());
}
}
else
{
files.emplace_back(relativePath.c_str());
}
}
}
closedir(dir);
}
return files;
}
#endif
} // namespace

169
tjp/core/file/Path.cpp Normal file
View File

@@ -0,0 +1,169 @@
// TJP COPYRIGHT HEADER
#ifdef TJP_CORE_HEADER_ONLY
#pragma once
#endif
#include "Path.h"
namespace tjp::core::file {
TJP_CORE_HEADER_ONLY_INLINE
std::string_view removeExtensions(const std::string_view &s)
{
auto lastSlash = s.rfind('/');
auto firstDotAfterLastSlash = s.find('.', lastSlash+1);
return s.substr(0, firstDotAfterLastSlash);
}
TJP_CORE_HEADER_ONLY_INLINE
std::string joinDirs(const std::vector<std::string_view> &dirs)
{
std::string joined;
for (auto &dir : dirs)
{
if (dir.empty())
continue;
bool first = joined.empty();
if (dir.front() == '/' && !first)
joined += dir.substr(1);
else
joined += dir;
if (joined.back() != '/')
joined.push_back('/');
}
return joined;
}
TJP_CORE_HEADER_ONLY_INLINE
std::string joinPaths(const std::vector<std::string_view> &dirs)
{
std::string joined;
for (auto &dir : dirs)
{
if (dir.empty())
continue;
bool first = joined.empty();
if (!first)
if (joined.back() != '/')
joined.push_back('/');
if (dir.front() == '/' && !first)
joined += dir.substr(1);
else
joined += dir;
}
return joined;
}
TJP_CORE_HEADER_ONLY_INLINE
std::string joinDirsAndFile(const std::vector<std::string_view> &dirs, const std::string &fileName)
{
return joinDirs(dirs) + fileName;
}
TJP_CORE_HEADER_ONLY_INLINE
std::string convertDelimiter (const std::string &str, char from, char to)
{
if (from==to) return str;
if (str.empty()) return str;
std::string copy = str;
for (auto &c : copy)
if (c == from)
c = to;
return copy;
}
TJP_CORE_HEADER_ONLY_INLINE
std::string convertDelimiter (const std::string &path)
{
return path;
}
template<typename T>
T popFileName_(const T &path)
{
auto lastDelimiter = path.rfind('/');
return path.substr(0, lastDelimiter+1);
}
template<typename T>
T popLast_(const T &path)
{
if (path.empty())
return {};
if (path.back() != '/')
return popFileName(path);
if (path.size() < 2)
return {};
auto lastDelimiter = path.rfind('/', path.size()-2);
return path.substr(0, lastDelimiter+1);
}
TJP_CORE_HEADER_ONLY_INLINE
std::string_view popFileName(const std::string_view &path)
{
return popFileName_(path);
}
TJP_CORE_HEADER_ONLY_INLINE
std::string_view popLast(const std::string_view &path)
{
return popLast_(path);
}
TJP_CORE_HEADER_ONLY_INLINE
std::string popFileName(const std::string &path)
{
return popFileName_(path);
}
TJP_CORE_HEADER_ONLY_INLINE
std::string popLast(const std::string &path)
{
return popLast_(path);
}
TJP_CORE_HEADER_ONLY_INLINE
std::string fileName(const std::string &path)
{
auto lastDelimiter = path.rfind('/');
if (lastDelimiter == std::string::npos)
return path;
return path.substr(lastDelimiter+1);
}
TJP_CORE_HEADER_ONLY_INLINE
std::string_view fileName(const std::string_view &path)
{
auto lastDelimiter = path.rfind('/');
if (lastDelimiter == std::string::npos)
return path;
return path.substr(lastDelimiter+1);
}
TJP_CORE_HEADER_ONLY_INLINE
std::string extension(const std::string &path)
{
auto lastDelimiter = path.rfind('.');
if (lastDelimiter == std::string::npos)
return "";
return path.substr(lastDelimiter);
}
} // namespace

44
tjp/core/file/Path.h Normal file
View File

@@ -0,0 +1,44 @@
// TJP COPYRIGHT HEADER
#pragma once
#include <vector>
#include <string>
#include <tjp/core/header_only/compile.h>
namespace tjp::core::file {
std::string joinDirs(const std::vector<std::string_view> &dirs);
std::string joinPaths(const std::vector<std::string_view> &paths);
std::string joinDirsAndFile(const std::vector<std::string_view> &dirs, const std::string_view &fileName);
std::string fileName(const std::string &path);
std::string_view fileName(const std::string_view &path);
std::string extension(const std::string &path);
std::string_view removeExtensions(const std::string_view &path);
std::string_view popFileName(const std::string_view &path);
std::string_view popLast(const std::string_view &path);
std::string popFileName(const std::string &path);
std::string popLast(const std::string &path);
#ifdef WIN32
const char LocalDirectoryDelimiter = '\\';
#else
const char LocalDirectoryDelimiter = '/';
#endif
const char ServerDirectoryDelimiter = '/';
std::string convertDelimiter (const std::string &path);
std::string convertDelimiter (const std::string &path, char from, char to);
} // namespace
#ifdef TJP_CORE_HEADER_ONLY
#include "Path.cpp"
#endif

31
tjp/core/file/VirtualFileSystem.cpp Normal file
View File

@@ -0,0 +1,31 @@
// TJP COPYRIGHT HEADER
#ifdef TJP_CORE_HEADER_ONLY
#pragma once
#endif
#include <tjp/core/header_only/compile.h>
#include "VirtualFileSystem.hpp"
namespace tjp::core::file {
TJP_CORE_HEADER_ONLY_INLINE
Contents readFile(const VirtualFileSystem &system, const StringView &path)
{
return system.get(path);
}
TJP_CORE_HEADER_ONLY_INLINE
void writeFile(VirtualFileSystem &system, const StringView &path, ContentsView contents)
{
return system.put(path, contents);
}
TJP_CORE_HEADER_ONLY_INLINE
bool fileExists(const VirtualFileSystem &system, const StringView &path)
{
return system.has(path);
}
} // namespace

20
tjp/core/file/VirtualFileSystem.h Normal file
View File

@@ -0,0 +1,20 @@
// TJP COPYRIGHT HEADER
#pragma once
#include "Contents.h"
#include <tjp/core/string/StringView.h>
namespace tjp::core::file {
struct VirtualFileSystem;
Contents readFile(const VirtualFileSystem &system, const StringView &path);
void writeFile(VirtualFileSystem &system, const StringView &path, ContentsView);
bool fileExists(const VirtualFileSystem &system, const StringView &path);
} // namespace
#ifdef TJP_CORE_HEADER_ONLY
#include "VirtualFileSystem.cpp"
#endif

108
tjp/core/file/VirtualFileSystem.hpp Normal file
View File

@@ -0,0 +1,108 @@
// TJP COPYRIGHT HEADER
#pragma once
#include "File.h"
#include "Path.h"
namespace tjp::core::file {
struct VirtualFileSystem
{
virtual ~VirtualFileSystem () {};
virtual String pathFor(const StringView &path) const = 0;
virtual Optional<Contents> get_if(const StringView &path) const = 0;
virtual Contents get(const StringView &path) const = 0;
virtual void put(const StringView &path, ContentsView) = 0;
virtual bool has(const StringView &path) const = 0;
} ;
struct VirtualFileSystemRoot : VirtualFileSystem
{
String base;
VirtualFileSystemRoot(const StringView &base_) :
base(base_)
{}
inline
String pathFor(const StringView &path) const override
{
return joinPaths({ base, String(path) });
}
inline
Optional<Contents> get_if(const StringView &path) const override
{
return file::read(pathFor(path));
}
inline
Contents get(const StringView &path) const override
{
return file::require_read(pathFor(path));
}
inline
void put(const StringView &path, ContentsView contents) override
{
file::ensurePath(base, path);
file::require_write(pathFor(path), contents);
}
inline
bool has(const StringView &path) const override
{
return file::fileExists(pathFor(path));
}
} ;
struct VirtualFileSystemRelative : VirtualFileSystem
{
VirtualFileSystem *to;
String base;
VirtualFileSystemRelative(VirtualFileSystem *to_, const StringView &base_) :
to(to_),
base(base_)
{}
String join(const StringView &path) const
{
return joinPaths({ base, String(path) });
}
String pathFor(const StringView &path) const override
{
return to->pathFor(join(path));
}
inline
Optional<Contents> get_if(const StringView &path) const override
{
return to->get_if(join(path));
}
inline
Contents get(const StringView &path) const override
{
return to->get(join(path));
}
inline
void put(const StringView &path, ContentsView contents) override
{
to->put(join(path), contents);
}
inline
bool has(const StringView &path) const override
{
return to->has(join(path));
}
} ;
} // namespace

32
tjp/core/file/_tests/Path.cpp Normal file
View File

@@ -0,0 +1,32 @@
// TJP COPYRIGHT HEADER
#include <tjp/core/testing/catch.hpp>
#include <tjp/core/containers/Vector.hpp>
#include <tjp/core/containers/Tuple.hpp>
#include <tjp/core/string/String.hpp>
#include "../Path.h"
namespace tjp::core::file {
namespace {
SCENARIO("core::file::Path")
{
GIVEN("fileNames")
{
Vector<Tuple<String, String>> tests {
{ "hello.cpp", "hello" },
{ "asdf.123.345.456", "asdf" },
{ "/asdf/123.asdf.sdf", "/asdf/123" },
{ "/asdf.zxc/123.asdf.sdf", "/asdf.zxc/123" }
};
for (auto &[t, e] : tests)
{
auto v = removeExtensions(t);
REQUIRE(v == e);
}
}
}
} // namespace
} // namespace

71
tjp/core/file/apple/Directories.cpp Normal file
View File

@@ -0,0 +1,71 @@
// TJP COPYRIGHT HEADER
#include "Directories.h"
#include "../Path.h"
#include <tjp/core/system/System.h>
#ifdef SYS_MAC
#include <mach-o/dyld.h>
#include "CoreFoundation/CFBundle.h"
#endif
#ifdef SYS_IOS
#include <mach-o/dyld.h>
#include "CoreFoundation/CFBundle.h"
#endif
#include <stdlib.h>
namespace tjp {
namespace core {
namespace file {
namespace apple {
std::string getHomeDirectory ()
{
const char *homeDir = getenv("HOME");
if (homeDir)
return homeDir;
return {};
}
std::string getExecutablePath ()
{
char path_[PATH_MAX+1];
uint32_t size=PATH_MAX;
_NSGetExecutablePath(path_, &size);
if (size > 0)
{
auto path = std::string(path_);
return path;
}
return {};
}
std::string getExecutableDirectory ()
{
auto path = getExecutablePath();
return popFileName(path);
}
std::string getExecutableBundleDirectory ()
{
return popLast(getExecutableDirectory());
}
std::string getExecutableResourceDirectory ()
{
auto path = getExecutableBundleDirectory();
return joinDirs({ path, "Resources"});
}
} // namespace
} // namespace
} // namespace
} // namespace

22
tjp/core/file/apple/Directories.h Normal file
View File

@@ -0,0 +1,22 @@
// TJP COPYRIGHT HEADER
#pragma once
#include <vector>
#include <string>
namespace tjp {
namespace core {
namespace file {
namespace apple {
std::string getExecutablePath();
std::string getExecutableDirectory();
std::string getExecutableBundleDirectory();
std::string getExecutableResourceDirectory();
std::string getHomeDirectory();
} // namespace
} // namespace
} // namespace
} // namespace

28
tjp/core/file/copy.cpp Normal file
View File

@@ -0,0 +1,28 @@
// TJP COPYRIGHT HEADER
#ifdef TJP_CORE_HEADER_ONLY
#pragma once
#endif
#include <tjp/core/header_only/compile.h>
#include "copy.hpp"
#include "read.hpp"
#include "write.hpp"
#include <cstdio>
namespace tjp::core::file {
TJP_CORE_HEADER_ONLY_INLINE
bool copy(const String &from, const String &to)
{
if (auto contents = read(from))
if (write(to, *contents))
return true;
return false;
}
} // namespace

15
tjp/core/file/copy.hpp Normal file
View File

@@ -0,0 +1,15 @@
// TJP COPYRIGHT HEADER
#pragma once
#include <tjp/core/string/String.hpp>
namespace tjp::core::file {
bool copy(const String &path, const String &to);
} // namespace
#ifdef TJP_CORE_HEADER_ONLY
#include "copy.cpp"
#endif

28
tjp/core/file/directory_exists.cpp Normal file
View File

@@ -0,0 +1,28 @@
// TJP COPYRIGHT HEADER
#ifdef TJP_CORE_HEADER_ONLY
#pragma once
#endif
#include <tjp/core/header_only/compile.h>
#include "directory_exists.hpp"
#include <sys/stat.h>
namespace tjp::core::file {
TJP_CORE_HEADER_ONLY_INLINE
bool directory_exists (const String &file)
{
struct stat status;
if (stat (file.c_str(), &status)==0)
{
// if it is not a directory return true
if (status.st_mode & S_IFDIR)
return true;
}
return false;
}
} // namespace

15
tjp/core/file/directory_exists.hpp Normal file
View File

@@ -0,0 +1,15 @@
// TJP COPYRIGHT HEADER
#pragma once
#include <tjp/core/string/String.hpp>
namespace tjp::core::file {
bool directory_exists(const String &path);
} // namespace
#ifdef TJP_CORE_HEADER_ONLY
#include "directory_exists.cpp"
#endif

20
tjp/core/file/directory_of.cpp Normal file
View File

@@ -0,0 +1,20 @@
// TJP COPYRIGHT HEADER
#ifdef TJP_HEADER_ONLY
#pragma once
#endif
#include <tjp/core/header_only/compile.h>
#include "directory_of.hpp"
namespace tjp::core::file {
TJP_CORE_HEADER_ONLY_INLINE
StringView directory_of(const StringView &path)
{
auto lastDelimiter = path.rfind('/');
return path.substr(0, lastDelimiter+1);
}
} // namespace

15
tjp/core/file/directory_of.hpp Normal file
View File

@@ -0,0 +1,15 @@
// TJP COPYRIGHT HEADER
#pragma once
#include <tjp/core/string/StringView.hpp>
namespace tjp::core::file {
StringView directory_of(const StringView &path);
} // namespace
#ifdef TJP_CORE_HEADER_ONLY
#include "directory_of.cpp"
#endif

34
tjp/core/file/file_exists.cpp Normal file
View File

@@ -0,0 +1,34 @@
// TJP COPYRIGHT HEADER
#ifdef TJP_CORE_HEADER_ONLY
#pragma once
#endif
#include <tjp/core/header_only/compile.h>
#include "file_exists.hpp"
#include <sys/stat.h>
namespace tjp::core::file {
TJP_CORE_HEADER_ONLY_INLINE
bool fileExists (const String &file)
{
struct stat status;
if (stat (file.c_str(), &status)==0)
{
// if it is not a directory return true
if (!(status.st_mode & S_IFDIR))
return true;
}
return false;
}
TJP_CORE_HEADER_ONLY_INLINE
bool fileExists (const StringView &file)
{
return fileExists(String(file));
}
} // namespace

17
tjp/core/file/file_exists.hpp Normal file
View File

@@ -0,0 +1,17 @@
// TJP COPYRIGHT HEADER
#pragma once
#include <tjp/core/string/String.hpp>
#include <tjp/core/string/StringView.hpp>
namespace tjp::core::file {
bool fileExists(const String &path);
bool fileExists(const StringView &path);
} // namespace
#ifdef TJP_CORE_HEADER_ONLY
#include "file_exists.cpp"
#endif

135
tjp/core/file/get_files.cpp Normal file
View File

@@ -0,0 +1,135 @@
// TJP COPYRIGHT HEADER
#ifdef TJP_HEADER_ONLY
#pragma once
#endif
#include <tjp/core/header_only/compile.h>
#include "get_files.hpp"
#include <tjp/core/system/System.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <dirent.h>
#include <sys/types.h>
#include <queue>
#include <filesystem>
namespace tjp::core::file {
#ifndef SYS_IOS
TJP_CORE_HEADER_ONLY_INLINE
Vector<String> get_files(const String &path, const Optional<String> &extension, bool recursive)
{
Vector<String> files;
try
{
std::queue<std::filesystem::path> dirs;
std::filesystem::path basePath = std::filesystem::canonical(path);
dirs.push(basePath);
while (!dirs.empty())
{
auto current = dirs.front();
dirs.pop();
for (const auto &entry : std::filesystem::directory_iterator(current))
{
if (entry.is_directory())
{
if (recursive)
dirs.push(entry.path());
}
else
if (entry.is_regular_file())
{
if (!extension || entry.path().extension() == *extension) {
files.push_back(std::filesystem::relative(entry.path(), basePath).string());
}
}
}
}
}
catch (std::filesystem::filesystem_error &)
{
}
catch (std::exception &)
{
}
catch (...)
{
}
return files;
}
#else
TJP_CORE_HEADER_ONLY_INLINE
Vector<String> get_files(const String &path_, const Optional<String> &extension, bool recursive)
{
auto path = String(path_);
Vector<String> files;
std::queue<std::string> dirs;
dirs.push("");
while (!dirs.empty())
{
auto relativeDir = dirs.front();
std::string absoluteDir = path + relativeDir;
dirs.pop();
DIR *dir = opendir(absoluteDir.c_str());
if (!dir) {
continue;
}
struct dirent *entry;
while ((entry = readdir(dir)) != nullptr)
{
std::string name(entry->d_name);
// skip "." and ".."
if (name == "." || name == "..") {
continue;
}
std::string relativePath =
relativeDir.empty() ?
name :
relativeDir + "/" + name;
if (entry->d_type == DT_DIR)
{
// enqueue subdirectory
if (recursive)
dirs.push(relativePath);
}
else
{
if (extension.has_value())
{
if (name.size() >= extension->size() &&
name.compare(name.size() - extension->size(), extension->size(), *extension) == 0)
{
files.emplace_back(relativePath.c_str());
}
}
else
{
files.emplace_back(relativePath.c_str());
}
}
}
closedir(dir);
}
return files;
}
#endif
} // namespace

Some files were not shown because too many files have changed in this diff Show More