flatten 20260225

tjp/core/internet/Data.hpp

@@ -0,0 +1,75 @@
+// TJP COPYRIGHT HEADER
+
+#pragma once
+
+#include <tjp/core/string/StringView.h>
+#include <tjp/core/containers/Vector.h>
+#include <tjp/core/containers/Optional.hpp>
+
+namespace tjp::core::internet {
+
+using DataBlock = Vector<char>;
+using DataView = StringView;
+
+struct Data {
+	mutable Optional<DataBlock> block;
+	mutable Optional<String> string;
+	
+	Data()
+	{}
+	
+	Data(DataBlock &&block_) :
+		block(std::move(block_))
+	{}
+	
+	Data(String &&string_) :
+		string(std::move(string_))
+	{}
+	
+	operator String &() const
+	{
+		if (!string.has_value())
+		{
+			if (block.has_value())
+				string = String(block->data(), block->size());
+			else
+				string = String();
+		}
+			
+		return *string;
+	}
+
+	operator DataView () const
+	{
+		if (block.has_value())
+			return StringView(block->data(), block->size());
+
+		if (string.has_value())
+			return *string;
+		
+		return {};
+	} ;
+	
+	const char *data() const
+	{
+		DataView view = *this;
+		return view.data();
+	}
+
+	const char *c_str() const
+	{
+		String &view = *this;
+		return view.c_str();
+	}
+	
+	const auto size() const
+	{
+		DataView view = *this;
+		return view.size();
+	}
+
+	
+};
+
+} // namespace
+

tjp/core/internet/Headers.cpp

@@ -0,0 +1,260 @@
+// TJP COPYRIGHT HEADER
+
+#ifdef TJP_CORE_HEADER_ONLY
+	#pragma once
+#endif
+#include <tjp/core/header_only/compile.h>
+
+#include "Headers.hpp"
+
+#include <iostream>
+#include <sstream>
+#include <iomanip>
+
+#include <cstring>
+
+#include <tjp/core/io/memstream.hpp>
+#include <tjp/core/string/from_string.hpp>
+#include <tjp/core/string/ToString.hpp>
+#include <tjp/core/string/cmp_case.hpp>
+
+namespace tjp::core::internet {
+
+TJP_CORE_HEADER_ONLY_INLINE
+const String *getHeader(const Headers &headers, StringView name)
+{
+	for (auto &[key, value]: headers)
+		if (core::cmp_case(key, name) == 0)
+			return &value;
+			
+	return nullptr;
+}
+
+TJP_CORE_HEADER_ONLY_INLINE
+Optional<std::time_t> parseDate(const String &s)
+{
+	std::tm tm = {};
+	core::io::imemstream ss(s.data(), s.size());
+
+	// Example format: "Fri, 02 Jun 2023 18:24:37 GMT"
+	ss >> std::get_time(&tm, "%a, %d %b %Y %H:%M:%S GMT");
+	if (ss.fail()) {
+		return {};
+	}
+
+	// Convert to time_t in UTC
+#if defined(_WIN32)
+	time_t t = _mkgmtime(&tm);
+#else
+	time_t t = timegm(&tm);
+#endif
+
+	return static_cast<std::time_t>(t);
+}
+
+TJP_CORE_HEADER_ONLY_INLINE
+Optional<std::time_t> getLastModified(const Headers &headers)
+{
+	if (auto value = getHeader(headers, "Last-Modified"))
+		return parseDate(*value);
+		
+	return {};
+}
+
+TJP_CORE_HEADER_ONLY_INLINE
+Optional<std::time_t> getDate(const Headers &headers)
+{
+	if (auto value = getHeader(headers, "Date"))
+		return parseDate(*value);
+		
+	return {};
+}
+
+TJP_CORE_HEADER_ONLY_INLINE
+Optional<String> getType(const Headers &headers)
+{
+	if (auto value = getHeader(headers, "Content-Type"))
+		return *value;
+		
+	return {};
+}
+
+TJP_CORE_HEADER_ONLY_INLINE
+Optional<String> getCacheControl(const Headers &headers)
+{
+	if (auto value = getHeader(headers, "Cache-Control"))
+		return *value;
+		
+	return {};
+}
+
+TJP_CORE_HEADER_ONLY_INLINE
+Optional<String> getContentLocation(const Headers &headers)
+{
+	if (auto value = getHeader(headers, "Content-Location"))
+		return *value;
+		
+	return {};
+}
+
+TJP_CORE_HEADER_ONLY_INLINE
+Optional<String> getLocation(const Headers &headers)
+{
+	if (auto value = getHeader(headers, "Location"))
+		return *value;
+		
+	return {};
+}
+
+TJP_CORE_HEADER_ONLY_INLINE
+Optional<String> getCookie(const Headers &headers)
+{
+	if (auto value = getHeader(headers, "Set-Cookie"))
+		return *value;
+		
+	return {};
+}
+
+TJP_CORE_HEADER_ONLY_INLINE
+Optional<s64> parseStatusCode(const String &s)
+{
+	if (s.rfind("HTTP/", 0) == 0)
+	{
+		int code = 0;
+		if (sscanf((char *)s.data(), "HTTP/%*s %d", &code) != 1)
+			return {};
+			
+		return code;
+	}
+	
+	return {};
+}
+
+
+TJP_CORE_HEADER_ONLY_INLINE
+Optional<Range> parseRange(const String &s)
+{
+	long long int start = -1, end = -1;
+
+	int matched = std::sscanf(s.data(), "bytes=%lld-%lld", &start, &end);
+	if (matched != 2)
+		return {};
+
+	return Range { start, end };
+}
+
+TJP_CORE_HEADER_ONLY_INLINE
+String makeRange(Range range)
+{
+	ToString string;
+	
+	auto [begin, end] = range;
+	string << "bytes=" << begin << '-' << end;
+	
+	return string;
+}
+
+TJP_CORE_HEADER_ONLY_INLINE
+Optional<ContentRange> parseContentRange(const String &s)
+{
+	char unit[16];
+	char totalStr[32];
+	long long int start = -1, end = -1, total = -1;
+
+	int matched = std::sscanf(s.data(), "%15s %lld-%lld/%31s", unit, &start, &end, totalStr);
+	if (matched != 4 || std::strcmp(unit, "bytes") != 0)
+		return {};
+
+	if (std::strcmp(totalStr, "*") == 0)
+		total = -1; // unknown total
+	else
+		total = std::atoll(totalStr);
+
+	return ContentRange { start, end, total };
+}
+
+TJP_CORE_HEADER_ONLY_INLINE
+Optional<s64> getCacheControlMaxAge(const String &s_)
+{
+	StringView s(s_);
+	const StringView prefix = "max-age=";
+	auto maxAgeP = s.find(prefix);
+	if (maxAgeP == s.npos)
+		return {};
+		
+	auto maxAgeB = maxAgeP + prefix.size();
+	auto semi = s.find(';', maxAgeB);
+	if (semi == s.npos)
+		semi = s.size();
+	
+	auto ageS = s.substr(maxAgeB, semi - maxAgeB);
+	return core::from_string<s64>(ageS);
+}
+
+
+TJP_CORE_HEADER_ONLY_INLINE
+String makeContentRange(ContentRange range)
+{
+	ToString string;
+	
+	auto [begin, end, of] = range;
+	string << "bytes " << begin << '-' << end << '/';
+	
+	if (of >= 0)
+		string << of;
+	else
+		string << '*';
+		
+	return string;
+}
+
+TJP_CORE_HEADER_ONLY_INLINE
+Optional<ContentRange> getContentRange(const Headers &headers)
+{
+	if (auto value = getHeader(headers, "Content-Range"))
+		return parseContentRange(*value);
+		
+	return {};
+}
+
+
+TJP_CORE_HEADER_ONLY_INLINE
+Optional<s64> parseContentSize(const String &s)
+{
+	long long int size = 0;
+	if (sscanf(s.c_str(), "%lld", &size) != 1)
+		return {};
+
+	return size;
+}
+
+TJP_CORE_HEADER_ONLY_INLINE
+Optional<s64> getContentSize(const Headers &headers)
+{
+	if (auto value = getHeader(headers, "Content-Length"))
+		return parseContentSize(*value);
+		
+	return {};
+}
+
+TJP_CORE_HEADER_ONLY_INLINE
+String makeDateHeader(std::time_t t)
+{
+	std::tm* gm = std::gmtime(&t); // convert to UTC
+	if (!gm) return {};
+
+	char buffer[128];
+	// Format: "Fri, 02 Jun 2023 18:24:37 GMT"
+	if (std::strftime(buffer, sizeof(buffer), "%a, %d %b %Y %H:%M:%S GMT", gm) == 0)
+		return {};
+
+	return String(buffer);
+}
+
+TJP_CORE_HEADER_ONLY_INLINE
+String makeLastModifiedHeader(std::time_t t)
+{
+	return makeDateHeader(t);
+}
+
+} // namespace

tjp/core/internet/Headers.hpp

@@ -0,0 +1,53 @@
+// TJP COPYRIGHT HEADER
+
+#pragma once
+
+#include <tjp/core/string/String.h>
+#include <tjp/core/containers/Tuple.h>
+#include <tjp/core/containers/Vector.h>
+#include <tjp/core/containers/Tuple.h>
+#include <tjp/core/containers/Optional.h>
+#include <tjp/core/string/StringView.h>
+#include <tjp/core/types/Types.h>
+
+#include <ctime>
+
+namespace tjp::core::internet {
+
+using Header = Tuple<String, String>;
+using Headers = Vector<Header>;
+
+const String *getHeader(const Headers &, StringView name);
+
+using ContentRange = Tuple<s64, s64, s64>;
+using Range = Tuple<s64, s64>;
+
+Optional<s64> parseStatusCode(const String &);
+Optional<std::time_t> parseDate(const String &);
+Optional<s64> parseContentSize(const String &);
+Optional<ContentRange> parseContentRange(const String &);
+Optional<Range> parseRange(const String &);
+
+Optional<String> getType(const Headers &);
+Optional<String> getCacheControl(const Headers &);
+Optional<String> getLocation(const Headers &);
+Optional<String> getContentLocation(const Headers &);
+Optional<String> getCookie(const Headers &);
+Optional<std::time_t> getDate(const Headers &);
+Optional<std::time_t> getLastModified(const Headers &);
+String makeDateHeader(std::time_t);
+String makeLastModifiedheader(std::time_t);
+
+Optional<s64> getContentSize(const Headers &);
+Optional<ContentRange> getContentRange(const Headers &);
+String makeContentRange(ContentRange);
+String makeRange(Range);
+
+Optional<s64> getCacheControlMaxAge(const String &);
+
+
+} // namespace
+
+#ifdef TJP_CORE_HEADER_ONLY
+	#include "Headers.cpp"
+#endif

tjp/core/internet/Internet+Internal.cpp

@@ -0,0 +1,147 @@
+// TJP COPYRIGHT HEADER
+
+
+#ifdef TJP_CORE_HEADER_ONLY
+	#pragma once
+#endif
+#include <tjp/core/header_only/compile.h>
+
+#include "Internet+Internal.hpp"
+#include <iostream>
+
+#include <tjp/core/log/Log.h>
+#include <tjp/core/log/LogOf.h>
+
+#include <cstring>
+
+namespace tjp::core::internet::internal {
+
+TJP_CORE_HEADER_ONLY_INLINE
+Request::~Request ()
+{
+	xDebugLine();
+}
+
+TJP_CORE_HEADER_ONLY_INLINE
+size_t receiveContentsCallback(void *ptr, size_t size, void *userdata)
+{
+	sLogDebug("core::internet::receive", logVar(size));
+
+	auto* progressiveData = static_cast<ProgressiveData*>(userdata);
+	if (auto request = strong(progressiveData->request))
+	{
+		auto &data = progressiveData->receive;
+		auto &options = progressiveData->options;
+		
+		auto realsize = size;
+		
+		char *ptr_ = (char *)ptr;
+		
+		if (options.onData)
+			options.onData(ptr_, realsize);
+		
+		data.data.data.insert(data.data.data.end(), ptr_, ptr_+realsize);
+		return realsize;
+	}
+	else
+	{
+		progressiveData->interrupted = true;
+	}
+	
+	sLogDebug("core::internet::receive", "lost ptr");
+	
+	return 0;
+}
+
+TJP_CORE_HEADER_ONLY_INLINE
+size_t sendContentsCallback(char *ptr, size_t size, void *userdata)
+{
+	sLogDebug("core::internet::send", logVar(size));
+
+	auto* progressiveData = static_cast<ProgressiveData*>(userdata);
+	if (auto request = strong(progressiveData->request))
+	{
+		auto &send = progressiveData->send;
+		auto &data = send.data;
+		auto &offset = send.offset;
+		
+		size_t buffer_size = size;
+		
+		size_t to_copy = std::min(data.size() - offset, buffer_size);
+		
+		std::memcpy(ptr, data.data() + offset, to_copy);
+		offset += to_copy;
+		
+		return to_copy;
+	}
+	else
+	{
+		progressiveData->interrupted = true;
+	}
+
+	sLogDebug("core::internet::send", "lost ptr");
+
+	return 0;
+}
+
+TJP_CORE_HEADER_ONLY_INLINE
+size_t receiveHeadersCallback(char *buffer, size_t size, void *userdata)
+{
+	sLogDebug("core::internet::headers", logVar(buffer) << logVar(size));
+
+	auto *progressiveData = static_cast<internal::ProgressiveData *>(userdata);
+	auto &options = progressiveData->options;
+
+	if (auto request = strong(progressiveData->request))
+	{
+		auto &receive = progressiveData->receive;
+		
+		StringView headerLine(buffer, size);
+		if (headerLine.rfind("HTTP/", 0) == 0)
+		{
+			int code = 0;
+			String s(buffer, size);
+			sscanf(s.c_str(), "HTTP/%*s %d", &code);
+			receive.data.statusCode = code;
+		}
+		else
+		if (headerLine == "\r\n")
+		{
+			if (options.onHeaders)
+			{
+				options.onHeaders(receive.data.headers, receive.data.statusCode);
+				options.onHeaders = {};
+			}
+		}
+		else
+		{
+			auto colon = headerLine.find(':');
+			if (colon != headerLine.npos)
+			{
+				auto key = headerLine.substr(0, colon);
+				auto value = headerLine.substr(colon+1);
+
+				while (!value.empty() && std::isspace(value.front()))
+					value.remove_prefix(1);
+					
+				while (!value.empty() && std::isspace(value.back()))
+					value.remove_suffix(1);
+				
+				receive.data.headers.push_back({ String(key), String(value)});
+			}
+		}
+		
+		return size;
+	}
+	else
+	{
+		progressiveData->interrupted = true;
+	}
+
+	sLogDebug("core::internet::headers", "lost ptr");
+
+	return 0;
+}
+
+} // namespace
+

tjp/core/internet/Internet+Internal.hpp

@@ -0,0 +1,61 @@
+// TJP COPYRIGHT HEADER
+
+#pragma once
+
+#include "Internet.hpp"
+#include <tjp/core/future/Future.hpp>
+
+namespace tjp::core::internet {
+
+namespace internal {
+
+enum class RequestType {
+	GET,
+	POST,
+	PUT,
+	HEAD
+} ;
+
+struct Request {
+	RequestType type;
+
+	String url;
+	HttpOptions options;
+	Optional<Data> data;
+	
+	Promise<FileContents> promise;
+	
+	~Request();
+} ;
+
+
+struct SendData
+{
+};
+
+struct ProgressiveData {
+	WeakPtr<Request> request;
+	HttpOptions options;
+	bool interrupted = false;
+	
+	struct {
+		FileContents data;
+	} receive;
+	
+	struct {
+		Data data;
+		size_t offset = 0;
+	} send;
+} ;
+
+size_t receiveContentsCallback(void *ptr, size_t size, void *receiveData);
+size_t receiveHeadersCallback(char *buffer, size_t size, void *userdata);
+size_t sendContentsCallback(char *ptr, size_t size, void *sendData);
+
+} // namespace
+
+} // namespace
+
+#ifdef TJP_CORE_HEADER_ONLY
+	#include "Internet+Internal.cpp"
+#endif

tjp/core/internet/Internet.h

@@ -0,0 +1,19 @@
+// TJP COPYRIGHT HEADER
+
+#pragma once
+
+#include <tjp/core/containers/Vector.h>
+#include <tjp/core/containers/Tuple.h>
+#include <tjp/core/string/String.h>
+
+namespace tjp::core::internet {
+
+using Header = Tuple<String, String>;
+using Headers = Vector<Header>;
+
+struct HttpOptions;
+struct FileContents;
+struct Exception;
+
+} // namespace
+

tjp/core/internet/Internet.hpp

@@ -0,0 +1,111 @@
+// TJP COPYRIGHT HEADER
+
+#pragma once
+
+#include "Internet.h"
+
+#include <string>
+#include <map>
+#include <vector>
+
+#include <tjp/core/future/Future.h>
+#include <tjp/core/ptr/Ptr.hpp>
+#include <tjp/core/string/String.h>
+#include <tjp/core/string/StringView.h>
+#include <tjp/core/containers/Tuple.h>
+#include <tjp/core/containers/Vector.h>
+#include <tjp/core/containers/Function.h>
+#include <tjp/core/types/Types.h>
+
+#include <tjp/core/exception/Exception.hpp>
+#include <tjp/core/containers/Optional.hpp>
+
+#include "Data.hpp"
+#include "Headers.hpp"
+
+#include <tjp/core/system/System.h>
+
+namespace tjp::core::internet {
+
+constexpr size_t GeneralFailure = 0xFFFF;
+
+using Header = Tuple<String, String>;
+using Headers = Vector<Header>;
+
+using StatusCode = s64;
+
+using HeadersCallback = Function<void(const Headers &, StatusCode code)>;
+using ProgressiveCallback = Function<void(char *, size_t)>;
+using CompleteCallback = Function<void(StatusCode)>;
+
+struct HttpOptions {
+	Headers headers;
+	time_t lastModifiedTime = 0;
+	time_t timeout = 0;
+	bool throwOnFailure = true;
+	size_t maxRedirects = 0;
+
+	Optional<String> httpVersion;
+	Optional<size_t> verbose;
+	
+	HeadersCallback onHeaders;
+	ProgressiveCallback onData;
+	CompleteCallback onComplete;
+} ;
+
+namespace HttpStatusCodes {
+	const size_t NotModified = 304;
+	const size_t OK = 200;
+} ;
+
+inline
+HttpOptions makeEmptyOptions() {
+	return {};
+}
+
+struct FileContents
+{
+	Headers headers;
+	DataBlock data;
+
+	StatusCode statusCode = GeneralFailure;
+	time_t lastModifiedTime = 0;
+	
+	bool ok() const
+	{
+		return statusCode < 400;
+	}
+} ;
+
+struct Exception : core::Exception {
+	using Super = core::Exception;
+	
+	Optional<FileContents> contents;
+	
+	Exception (
+		const String &what_,
+		Optional<FileContents> &&contents_
+	) :
+		Super(what_),
+		contents(std::move(contents_))
+	{
+	}
+	
+	const Optional<FileContents> &getContents() const {
+		return contents;
+	}
+} ;
+
+Future<FileContents> get(const std::string &url, const HttpOptions &o = makeEmptyOptions());
+Future<FileContents> post(const std::string &url, Data &&, const HttpOptions &o = makeEmptyOptions());
+Future<FileContents> put(const std::string &url, Data &&, const HttpOptions &o = makeEmptyOptions());
+
+} // namespace
+
+#ifdef TJP_CORE_HEADER_ONLY
+	#if defined(SYS_IOS)
+		#include "ios/Internet.hpp"
+	#else
+		#include "curl/Internet.hpp"
+	#endif
+#endif

tjp/core/internet/Queue.h

@@ -0,0 +1,15 @@
+// TJP COPYRIGHT HEADER
+
+#pragma once
+
+namespace tjp::core::internet {
+
+struct Queue;
+
+struct MainQueue;
+
+template<typename T>
+struct Queues;
+
+} // namespace
+

tjp/core/internet/Queue.hpp

@@ -0,0 +1,89 @@
+// TJP COPYRIGHT HEADER
+
+#pragma once
+
+#include "Internet.hpp"
+#include "Internet+Internal.hpp"
+#include <tjp/core/ptr/Ptr.hpp>
+#include <tjp/core/threads/ThreadSafe.hpp>
+#include <tjp/core/future/Future.hpp>
+#include <tjp/core/string/String.hpp>
+#include <tjp/core/containers/Optional.hpp>
+
+#include <queue>
+
+namespace tjp::core::internet {
+
+struct Queue
+{
+	struct Internal;
+	StrongPtr<Internal> i;
+	
+	Queue();
+	~Queue();
+
+	void prepare();
+
+	Future<FileContents> execute(internal::RequestType, const String &url, Data &&data, const HttpOptions &o = makeEmptyOptions());
+
+	Future<FileContents> get(const String &url, const HttpOptions &o = makeEmptyOptions())
+	{
+		return execute(internal::RequestType::GET, url, {}, o);
+	}
+	
+	Future<FileContents> post(const std::string &url, Data &&data, const HttpOptions &o = makeEmptyOptions())
+	{
+		return execute(internal::RequestType::POST, url, std::move(data), o);
+	}
+
+	Future<FileContents> put(const std::string &url, Data &&data, const HttpOptions &o = makeEmptyOptions())
+	{
+		return execute(internal::RequestType::PUT, url, std::move(data), o);
+	}
+
+	Future<FileContents> head(const std::string &url, const HttpOptions &o = makeEmptyOptions())
+	{
+		return execute(internal::RequestType::HEAD, url, {}, o);
+	}
+} ;
+
+struct MainQueue {};
+
+template<typename T>
+struct Queues {
+	static Queue queue;
+	
+	static Future<FileContents> get(const String &url, const HttpOptions &o = makeEmptyOptions())
+	{
+		return queue.execute(internal::RequestType::GET, url, {}, o);
+	}
+	
+	static Future<FileContents> post(const std::string &url, Data &&data, const HttpOptions &o = makeEmptyOptions())
+	{
+		return queue.execute(internal::RequestType::POST, url, std::move(data), o);
+	}
+
+	static Future<FileContents> put(const std::string &url, Data &&data, const HttpOptions &o = makeEmptyOptions())
+	{
+		return queue.execute(internal::RequestType::PUT, url, std::move(data), o);
+	}
+
+	static Future<FileContents> head(const std::string &url, const HttpOptions &o = makeEmptyOptions())
+	{
+		return queue.execute(internal::RequestType::HEAD, url, {}, o);
+	}
+} ;
+
+template<typename T>
+Queue Queues<T>::queue;
+
+} // namespace
+
+
+#ifdef TJP_CORE_HEADER_ONLY
+	#if defined(SYS_IOS)
+		#include "ios/Queue.hpp"
+	#else
+		#include "curl/Queue.hpp"
+	#endif
+#endif

tjp/core/internet/URL.cpp

@@ -0,0 +1,207 @@
+// TJP COPYRIGHT HEADER
+
+// TJP	4/09/01	relative URLs have not been checked yet.  don't expect them
+//		to work the first time you use this class!
+//
+// TJP	4/11/01	getAsLocalPath needs to be modified to work with all sorts
+//		of protocols and port numbers... right now, probably not
+
+#include "URL.h"
+
+#include "../file/Path.h"
+#include <tjp/core/string/cmp_case.hpp>
+#include <iostream>
+
+namespace tjp {
+namespace core {
+namespace internet {
+
+const URL Empty;
+
+const char FS = '/';
+//const char BS = '\\';
+const char *DD = "..";
+
+bool URL::isAbsoluteURL (const std::string &url)
+{
+	return url.find("://") != -1;
+}
+
+void URL::parseAbsoluteURL (const std::string &url)
+{
+	auto protocolBegin = 0;
+	auto protocolEnd = url.find("://");
+
+	if (protocolEnd==-1)
+	{
+		throw Exception ("Could not find protocol");
+	}
+
+	auto serverBegin = protocolEnd + std::string("://").length();
+	auto serverEnd = url.find (FS, serverBegin);
+
+	if (serverEnd==-1)
+	{
+		throw Exception ("Could not find server");
+	}
+	
+	auto directoryBegin = serverEnd + 1;
+	auto directoryEnd = url.find_last_of (FS) + 1;
+
+	auto fileBegin = directoryEnd;
+	auto fileEnd = url.length();
+
+	protocol = url.substr (protocolBegin, protocolEnd-protocolBegin);
+
+	server = url.substr (serverBegin, serverEnd-serverBegin);
+	directory = url.substr (directoryBegin, directoryEnd-directoryBegin);
+	file = url.substr (fileBegin, fileEnd-fileBegin);
+}
+
+void URL::parseRelativeURL (const std::string &url)
+{
+	auto directoryBegin = 0;
+	auto directoryEnd = url.find_last_of (FS) + 1;
+
+	auto fileBegin = directoryEnd;
+	auto fileEnd = url.length();
+
+	std::string relativeDirectory = url.substr (directoryBegin, directoryEnd-directoryBegin);
+
+	if (relativeDirectory.substr (0,2) == "./")
+	{
+		relativeDirectory = relativeDirectory.substr(2);
+	}
+
+	std::string relativeFile = url.substr (fileBegin, fileEnd-fileBegin);
+
+	if ( (relativeDirectory.length()>0 && directory.length()>0) &&
+		 ((directory[directory.length()-1]==FS) && relativeDirectory[0]==FS) )
+	{
+		directory += relativeDirectory.substr(1);
+	}
+	else
+	{
+		directory += relativeDirectory;
+	}
+
+	file = url.substr (fileBegin, fileEnd-fileBegin);
+}
+
+URL::URL ()
+{
+}
+
+URL::URL (const std::string &url)
+{
+	parseAbsoluteURL (url);
+}
+
+URL::URL (const URL &parent, const std::string &url)
+{
+	if (isAbsoluteURL (url))
+	{
+		parseAbsoluteURL (url);
+	}
+	else
+	{
+		protocol = parent.getProtocol();
+		server = parent.getServer();
+		directory = parent.getDirectory();
+
+		parseRelativeURL (url);
+	}
+}
+
+URL::URL (const URL &url)
+{
+	protocol = url.protocol;
+	server = url.server;
+	directory = url.directory;
+	file = url.file;
+}
+
+URL::~URL ()
+{
+}
+
+const std::string &URL::getProtocol () const
+{
+	return protocol;
+}
+
+const std::string &URL::getServer() const
+{
+	return server;
+}
+
+std::string URL::getPath () const
+{
+	return directory + file;
+}
+
+const std::string &URL::getDirectory () const
+{
+	return directory;
+}
+
+const std::string &URL::getFile () const
+{
+	return file;
+}
+
+URL URL::join(const URL &url, const std::string &append)
+{
+	if (append.empty())
+		return url;
+
+	return URL((std::string)url + FS + append);
+}
+
+URL::operator std::string() const
+{
+	return getProtocol() + "://" + getServer() + FS + getPath();
+}
+
+bool URL::operator <(const URL &right) const
+{
+	// this implementation should be redone probably
+
+	auto p = cmp_case(protocol, right.protocol);
+	if (p < 0) return true;
+	if (p > 0) return false;
+	
+	auto s = cmp_case(server, right.server);
+	if (s < 0) return true;
+	if (s > 0) return false;
+	
+	auto d = cmp_case(directory, right.directory);
+	if (d < 0) return true;
+	if (d > 0) return false;
+	
+	return less_case(file, right.file);
+}
+
+bool URL::operator ==(const URL &right) const
+{
+	return (
+		equal_case(protocol, right.protocol) &&
+		equal_case(server, right.server) &&
+		equal_case(directory, right.directory) &&
+		equal_case(file, right.file)
+	);
+}
+
+bool URL::operator !=(const URL &right) const
+{
+	return (!(*this == right));
+}
+
+std::ostream &operator <<(std::ostream &out, const URL &url)
+{
+	return out << (std::string)url;
+}
+
+} // namespace
+} // namespace
+} // namespace

tjp/core/internet/URL.h

@@ -0,0 +1,130 @@
+// TJP COPYRIGHT HEADER
+
+#pragma once
+
+#include <string>
+#include <tjp/core/exception/Exception.hpp>
+
+namespace tjp {
+namespace core {
+namespace internet {
+
+/**
+ * a URL is defined as a protocal specifying string + a server string + a directory
+ * + a (optional) file.  Such as ftp://myserver/mydirectory/myfile.
+ *
+ * For the purposes of this application a URL may be created using a parent URL, in
+ * which case, if the protocal is NOT specified, the URL is assumed to be a RELATIVE
+ * URL.  In this case, only the directory and file are used.. So, the URL:
+ * "/directory/myfile" created with the parent URL - "ftp://myserver/mydirectory"
+ * on construction would expand to: "ftp://myserver/mydirectory/directory/myfile"
+ *
+ * I am not sure whether this is exactly the standard or not, perhaps it should
+ * be modified later.
+ */
+class URL
+{
+	public:
+		/**
+		 * The single exception thrown by the url class on errors
+		 */
+		DECLARE_EXCEPTION (Exception);
+
+	protected:
+		std::string protocol;
+		std::string server;
+		std::string directory;
+		std::string file;
+
+		bool isAbsoluteURL (const std::string &);
+		void parseAbsoluteURL (const std::string &);
+		void parseRelativeURL (const std::string &);
+
+	public:
+		/**
+		 * default constructor
+		 */
+		URL ();
+
+		/**
+		 * throws an exception if the protocal is unknown, or if
+		 * the path somehow is invalid, such as "ftp://me.com/../SystemData"
+		 * because that dot.dot my cause the resource to point to our files.
+		 */
+		URL (const std::string &);
+
+		/**
+		 * when given a parent URL, if the child URL does not have a
+		 * protocal or server it is interpreted as a relative URL
+		 *
+		 * relative URL's are resolved on construction, pointers are not
+		 * kept hanging around
+		 */
+		URL (const URL &parent, const std::string &);
+
+		/**
+		 * copy constructor
+		 */
+		URL (const URL &);
+
+		/**
+		 * destructor
+		 */
+		virtual ~URL ();
+
+		/**
+		 * returns the string name of the protocal in string form
+		 */
+		const std::string &getProtocol () const;
+
+		/**
+		 * returns the server segment on the string (server:port)
+		 */
+		const std::string &getServer() const;
+
+		/**
+		 * returns the file segment of the string (file_path)
+		 */
+		std::string getPath () const;
+
+		/**
+		 * returns the directory portion of the path
+		 */
+		const std::string &getDirectory () const;
+
+		/**
+		 * returns the file portion of the path
+		 */
+		const std::string &getFile () const;
+
+		/**
+		 * returns the file portion of the path
+		 */
+		static URL join(const URL &, const std::string &);
+
+		/**
+		 * converts the URL into a string
+		 */
+		operator std::string() const;
+
+		/**
+		 * comparison operators for container classes
+		 */
+		bool operator <(const URL &right) const;
+		bool operator ==(const URL &right) const;
+		bool operator !=(const URL &right) const;
+
+		//==========================================================
+
+		/**
+		 * for comparisons
+		 */
+		static const URL Empty;
+} ;
+
+std::ostream &operator <<(std::ostream &, const URL &);
+
+} // namespace utilities
+} // namespace utilities
+} // namespace
+

tjp/core/internet/_tests/Internet+common.cpp

@@ -0,0 +1,21 @@
+// TJP COPYRIGHT HEADER
+
+#include "../join.hpp"
+#include <tjp/core/testing/catch.hpp>
+
+namespace tjp::core::internet::test {
+
+SCENARIO("core::internet")
+{
+	GIVEN("join")
+	{
+		REQUIRE(join("a/", "/b") == "a/b");
+		REQUIRE(join("a", "b") == "a/b");
+		REQUIRE(join("a/", "b") == "a/b");
+		REQUIRE(join("a", "/b") == "a/b");
+		REQUIRE(join("a", "") == "a");
+		REQUIRE(join("", "b") == "b");
+	}
+}
+
+} // namespace

tjp/core/internet/curl/Internet.cpp

@@ -0,0 +1,289 @@
+// TJP COPYRIGHT HEADER
+
+#include <tjp/core/header_only/compile.h>
+
+#ifdef TJP_CORE_HEADER_ONLY
+	#pragma once
+#endif
+
+#include "Internet.hpp"
+
+#include <curl/curl.h>
+#include <string.h>
+#include <stdlib.h>
+#include <string>
+#include <cstdlib>
+
+#include <tjp/core/string/String.hpp>
+#include <tjp/core/string/as_string_view.hpp>
+#include <tjp/core/algorithm/ExecuteOnDestruct.hpp>
+
+#include <tjp/core/future/Future.hpp>
+#include <tjp/core/log/Log.h>
+#include <tjp/core/log/LogOf.h>
+
+
+//===============================================================================
+
+namespace tjp {
+namespace core {
+namespace internet {
+
+TJP_CORE_HEADER_ONLY_INLINE
+size_t receiveContentsCallback(void *ptr, size_t size, size_t nmemb, void *userdata)
+{
+	return internal::receiveContentsCallback(ptr, size*nmemb, userdata);
+}
+
+TJP_CORE_HEADER_ONLY_INLINE
+size_t sendContentsCallback(char *ptr, size_t size, size_t nmemb, void *userdata)
+{
+	return internal::sendContentsCallback(ptr, size*nmemb, userdata);
+}
+
+TJP_CORE_HEADER_ONLY_INLINE
+size_t receiveHeadersCallback(char *ptr, size_t size, size_t nitems, void *userdata)
+{
+	return internal::receiveHeadersCallback(ptr, size*nitems, userdata);
+	
+}
+
+TJP_CORE_HEADER_ONLY_INLINE
+Future<FileContents> get (const std::string &url, const HttpOptions &options)
+{
+	auto *connection = curl_easy_init();
+
+	ExecuteOnDestruct cleanup = [&]() {
+		curl_easy_cleanup(connection);
+	};
+	
+	auto request = strong<internal::Request>();
+	request->type = internal::RequestType::GET;
+	request->url = url;
+	request->options = options;
+	
+	perform(request, connection);
+	
+	return request->promise.get_future();
+}
+
+TJP_CORE_HEADER_ONLY_INLINE
+Future<FileContents> post (const std::string &url, Data &&data, const HttpOptions &options)
+{
+	auto *connection = curl_easy_init();
+
+	ExecuteOnDestruct cleanup = [&]() {
+		curl_easy_cleanup(connection);
+	};
+	
+	auto request = strong<internal::Request>();
+	request->type = internal::RequestType::POST;
+	request->url = url;
+	request->options = options;
+	request->data = std::move(data);
+	
+	perform(request, connection);
+	
+	return request->promise.get_future();
+}
+
+TJP_CORE_HEADER_ONLY_INLINE
+void perform (const WeakPtr<internal::Request> &request_, CURL *connection)
+{
+	sLogDebug("core::internet", "perform");
+
+	curl_easy_reset(connection);
+	
+	auto getFile = [](const WeakPtr<internal::Request> &request_, CURL *connection) {
+
+		internal::ProgressiveData data;
+		data.request = request_;
+		String url;
+
+		struct curl_slist* headers = NULL;
+
+		if (auto request = strong(request_))
+		{
+			url = request->url;
+			data.options = std::move(request->options);
+			if (request->data)
+				data.send.data = std::move(*request->data);
+			
+			sLogDebug("core::internet", "starting " << logVar(url));
+
+			FileContents contents;
+			
+			if (!connection)
+				return contents;
+
+			CURLU *u = curl_url();
+			char *user_ = nullptr;
+			char *password_ = nullptr;
+			Optional<String> user, password;
+
+			if (curl_url_set(u, CURLUPART_URL, url.c_str(), 0) == CURLUE_OK)
+			{
+				if (curl_url_get(u, CURLUPART_USER, &user_, 0) == CURLUE_OK)
+					user = user_;
+					
+				if (curl_url_get(u, CURLUPART_PASSWORD, &password_, 0) == CURLUE_OK)
+					password = password_;
+
+			}
+			curl_url_cleanup(u);
+
+			for (auto &header: data.options.headers)
+			{
+				auto &[key, value] = header;
+				sLogDebug("core::internet::headers", "send header " << logVar(key) << logVar(value));
+
+				headers = curl_slist_append(headers, (key + ": " + value).c_str());
+			}
+
+			curl_easy_setopt (connection, CURLOPT_HTTPHEADER, headers);
+			
+			curl_easy_setopt(connection, CURLOPT_SSL_VERIFYPEER, false);
+			
+	//		if (CA_BUNDLE_PATH)
+	//			curl_easy_setopt(connection, CURLOPT_CAINFO, CA_BUNDLE_PATH);
+				
+			curl_easy_setopt (connection, CURLOPT_USERAGENT, "User-Agent: Curl Client");
+//			curl_easy_setopt (connection, CURLOPT_ACCEPT_ENCODING, "gzip, deflate");
+//			curl_easy_setopt (connection, CURLOPT_ACCEPT_ENCODING, "");
+			curl_easy_setopt (connection, CURLOPT_URL, url.c_str());
+			curl_easy_setopt (connection, CURLOPT_WRITEFUNCTION, receiveContentsCallback);
+			curl_easy_setopt (connection, CURLOPT_WRITEDATA, (void *)&data);
+			curl_easy_setopt (connection, CURLOPT_HEADERFUNCTION, receiveHeadersCallback);
+			curl_easy_setopt (connection, CURLOPT_HEADERDATA, (void *)&data);
+			
+			if (user)
+				curl_easy_setopt(connection, CURLOPT_USERNAME, user->c_str());
+			if (password)
+				curl_easy_setopt(connection, CURLOPT_PASSWORD, password->c_str());
+				
+			curl_easy_setopt(connection, CURLOPT_HTTPAUTH, CURLAUTH_BASIC);
+			
+			if (data.options.httpVersion == "2")
+				curl_easy_setopt(connection, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
+			
+			if (data.options.maxRedirects)
+			{
+				curl_easy_setopt(connection, CURLOPT_FOLLOWLOCATION, 1L);
+				curl_easy_setopt(connection, CURLOPT_MAXREDIRS, data.options.maxRedirects);
+				curl_easy_setopt(connection, CURLOPT_POSTREDIR, CURL_REDIR_POST_ALL);
+			}
+
+			if (request->type == internal::RequestType::GET)
+			{
+				curl_easy_setopt(connection, CURLOPT_HTTPGET, 1L);
+			}
+			else
+			if (request->type == internal::RequestType::HEAD)
+			{
+				curl_easy_setopt(connection, CURLOPT_NOBODY, 1);
+			}
+			else
+			if (request->type == internal::RequestType::POST)
+			{
+				DataView block = data.send.data;
+				curl_easy_setopt(connection, CURLOPT_POST, 1);
+				curl_easy_setopt(connection, CURLOPT_POSTFIELDS, block.data());
+				curl_easy_setopt(connection, CURLOPT_POSTFIELDSIZE_LARGE, block.size());
+			}
+			else
+			if (request->type == internal::RequestType::PUT)
+			{
+				curl_easy_setopt(connection, CURLOPT_UPLOAD, 1);
+				curl_easy_setopt(connection, CURLOPT_READDATA, &data);
+				curl_easy_setopt(connection, CURLOPT_READFUNCTION, sendContentsCallback);
+				curl_easy_setopt(connection, CURLOPT_INFILESIZE_LARGE, (curl_off_t)data.send.data.size());
+			}
+			
+			if (data.options.timeout)
+				curl_easy_setopt(connection, CURLOPT_TIMEOUT, data.options.timeout);
+
+		//	curl_easy_setopt (connection, CURLOPT_DNS_CACHE_TIMEOUT, 0L);
+		//	curl_easy_setopt (connection, CURLOPT_NOSIGNAL, 1L);
+		//	curl_easy_setopt (connection, CURLOPT_DNS_USE_GLOBAL_CACHE, 0L);
+			curl_easy_setopt (connection, CURLOPT_FILETIME, 1L);
+			
+			if (data.options.verbose)
+				curl_easy_setopt(connection, CURLOPT_VERBOSE, 1L);
+
+		}
+		else
+		{
+			throw Exception (
+				"HTTPConnection canceled",
+				{}
+			);
+		}
+		
+		StatusCode code;
+		CURLcode res = curl_easy_perform(connection);
+		curl_slist_free_all(headers);
+		
+		if (res != CURLE_OK)
+		{
+			data.receive.data.statusCode = GeneralFailure;
+		}
+		else
+		{
+			curl_easy_getinfo (connection, CURLINFO_RESPONSE_CODE, &code);
+			data.receive.data.statusCode = code;
+			xLogDebug (logVar(code));
+		}
+
+		StatusCode lastModifiedTime;
+		curl_easy_getinfo(connection, CURLINFO_FILETIME, &lastModifiedTime);
+		data.receive.data.lastModifiedTime = lastModifiedTime;
+	
+		if (data.receive.data.statusCode < 0)
+			data.receive.data.statusCode = GeneralFailure-1;
+
+		if (data.interrupted)
+			data.receive.data.statusCode = GeneralFailure-2;
+
+		sLogDebugIf(res, "debug", "finishing " << logVar(url) << logVar(res) << logVar(data.receive.data.statusCode));
+	
+		if (data.options.onHeaders)
+			data.options.onHeaders(data.receive.data.headers, (int)data.receive.data.statusCode);
+
+		if (data.options.onComplete)
+			data.options.onComplete(data.receive.data.statusCode);
+	
+		sLogDebugIf(res, "debug", "finished " << logVar(url) << logVar(res) << logVar(data.receive.data.statusCode));
+
+		if (res > 0)
+		{
+			sLogRelease("errors", "get http failed " << logVar(url) << logVar(res));
+			throw Exception (
+				"HTTPConnection getFile failed " + std::to_string(res),
+				std::move(data.receive.data)
+			);
+		}
+		if (data.options.throwOnFailure && !data.receive.data.ok())
+		{
+			sLogRelease("errors", "get http failed " << logVar(url) << logVar(data.receive.data.statusCode));
+			throw Exception (
+				"HTTPConnection getFile failed code " + std::to_string(data.receive.data.statusCode),
+				std::move(data.receive.data)
+			);
+		}
+
+		return data.receive.data;
+	};
+	
+	Promise<FileContents> promise;
+	if (auto request = strong(request_))
+	{
+		sLogDebug("core::internet", "perform will consume");
+		promise = std::move(request->promise);
+	}
+
+	promise.consume(getFile, request_, connection);
+}
+
+} // namespace
+} // namespace
+} // namespace

tjp/core/internet/curl/Internet.hpp

@@ -0,0 +1,17 @@
+// TJP COPYRIGHT HEADER
+
+#pragma once
+
+#include "../Internet.hpp"
+#include "../Internet+Internal.hpp"
+
+namespace tjp::core::internet {
+
+typedef void CURL;
+void perform (const WeakPtr<internal::Request> &request, CURL *connection);
+
+} // namespace
+
+#ifdef TJP_CORE_HEADER_ONLY
+	#include "Internet.cpp"
+#endif

tjp/core/internet/curl/Queue.cpp

@@ -0,0 +1,126 @@
+// TJP COPYRIGHT HEADER
+
+#ifdef TJP_CORE_HEADER_ONLY
+	#pragma once
+#endif
+
+#include "../Queue.hpp"
+#include "Internet.hpp"
+#include <curl/curl.h>
+
+#include <tjp/core/threads/Lock.hpp>
+#include <tjp/core/threads/process_queue.hpp>
+#include <thread>
+
+#include <tjp/core/ptr/strong_of.hpp>
+#include <tjp/core/future/Future.hpp>
+
+#include <queue>
+
+//===============================================================================
+
+namespace tjp::core::internet {
+
+struct Queue::Internal
+{
+	std::thread thread;
+	bool finished = false;
+
+	std::queue<WeakPtr<internal::Request>> queue;
+	Event event;
+	Event::Mutex mutex;
+	
+	CURL *curl;
+	
+	void run ();
+	void runOne (const WeakPtr<internal::Request> &i);
+	
+	Internal();
+	~Internal();
+} ;
+
+TJP_CORE_HEADER_ONLY_INLINE
+Queue::Queue()
+{
+}
+
+TJP_CORE_HEADER_ONLY_INLINE
+Queue::~Queue ()
+{
+}
+
+TJP_CORE_HEADER_ONLY_INLINE
+Queue::Internal::Internal()
+{
+	curl = curl_easy_init();
+}
+
+TJP_CORE_HEADER_ONLY_INLINE
+Queue::Internal::~Internal()
+{
+	curl_easy_cleanup(curl);
+}
+
+TJP_CORE_HEADER_ONLY_INLINE
+void Queue::Internal::runOne (const WeakPtr<internal::Request> &i)
+{
+	internet::perform(i, curl);
+}
+
+TJP_CORE_HEADER_ONLY_INLINE
+void Queue::Internal::run ()
+{
+	threads::process_queue(
+		finished,
+		event,
+		mutex,
+		queue,
+		[this](auto &i) { return runOne(i); }
+	);
+}
+
+TJP_CORE_HEADER_ONLY_INLINE
+void Queue::prepare()
+{
+	if (i)
+		return;
+		
+	static Mutex mutex;
+	auto lock = lock_of(mutex);
+	if (i)
+		return;
+		
+	i = strong<Internal>();
+	i->thread = std::thread([s=i](){ s->run(); });
+}
+
+TJP_CORE_HEADER_ONLY_INLINE
+Future<FileContents> Queue::execute(internal::RequestType type, const std::string &url, Data &&data, const HttpOptions &options)
+{
+	prepare();
+
+	auto item = strong_of(
+		internal::Request {
+			.type = type,
+			.url = url,
+			.options = options,
+			.data = std::move(data)
+		}
+	);
+		
+	{
+		auto lock = lock_of(i->mutex);
+		i->queue.push(weak(item));
+	}
+	
+	i->event.notify_all();
+	
+	return item->promise.get_future().then(
+		[item](auto &&f) {
+			return f.get();
+		}
+	);
+	
+}
+
+} // namespace

tjp/core/internet/curl/Queue.hpp

@@ -0,0 +1,7 @@
+// TJP COPYRIGHT HEADER
+
+#pragma once
+
+#ifdef TJP_CORE_HEADER_ONLY
+	#include "Queue.cpp"
+#endif

tjp/core/internet/form_encode.hpp

@@ -0,0 +1,49 @@
+// TJP COPYRIGHT HEADER
+
+#pragma once
+
+#include <tjp/core/string/String.h>
+#include <tjp/core/string/StringView+concat.hpp>
+#include <tjp/core/containers/Map.h>
+#include "url_param_encode.hpp"
+
+
+namespace tjp::core::internet {
+
+
+template<typename T>
+void form_encode_(String &result, T &&t)
+{
+	bool isKey = (result.empty() || result.back() != '=');
+	
+	if (isKey && !result.empty())
+		result += '&';
+		
+	result += url_param_encode(std::forward<T>(t));
+	
+	if (isKey)
+		result += '=';
+}
+
+template<typename ... Args>
+String form_encode(Args && ...args)
+{
+	String result;
+	(form_encode_(result, std::forward<Args>(args)), ...);
+	
+	return result;
+}
+
+inline
+String form_encode_map(const Map<String, String> &args)
+{
+	String result;
+	for (auto &[key, value]: args)
+	{
+		form_encode_(result, key);
+		form_encode_(result, value);
+	}
+	
+	return result;
+}
+} // namespace

tjp/core/internet/future_json_to_result.hpp

@@ -0,0 +1,18 @@
+// TJP COPYRIGHT HEADER
+
+#pragma once
+
+#include <tjp/core/io/json/in_sajson.h>
+
+namespace tjp::core::internet {
+
+template<typename T>
+auto future_json_to_result()
+{
+	return [](auto &&f) {
+		auto f_ = f.get();
+		return core::io::json::deserialize<T>(f_.data);
+	};
+}
+
+} // namespace

tjp/core/internet/get_embedded_value.hpp

@@ -0,0 +1,27 @@
+// TJP COPYRIGHT HEADER
+
+#pragma once
+
+#include "Internet.hpp"
+#include <tjp/core/string/as_string_view.hpp>
+
+namespace tjp::core::internet {
+
+inline
+Optional<StringView> get_embedded_value(const FileContents &value, const StringView &prefix, const StringView &postfix)
+{
+	auto s = as_string_view(value.data);
+	
+	auto p = s.find(prefix);
+	if (p == -1)
+		return {};
+		
+	s = s.substr(p + prefix.size());
+	auto pe = s.find(postfix);
+	if (pe == -1)
+		return {};
+		
+	return StringView(s.substr(0, pe));
+}
+
+} // namespace

tjp/core/internet/get_redirect.hpp

@@ -0,0 +1,25 @@
+// TJP COPYRIGHT HEADER
+
+#pragma once
+
+#include "Internet.hpp"
+#include <tjp/core/string/cmp_case.hpp>
+
+namespace tjp::core::internet {
+
+inline
+Optional<String> get_redirect(const FileContents &value)
+{
+	for (auto &header: value.headers)
+	{
+		auto &[key, value] = header;
+		if (cmp_case(key, "location") == 0)
+		{
+			return value;
+		}
+	}
+	
+	return {};
+}
+
+} // namespace

tjp/core/internet/hirose/Internet.cpp

@@ -0,0 +1,141 @@
+// TJP COPYRIGHT HEADER
+
+#include <tjp/core/header_only/compile.h>
+
+#ifdef TJP_CORE_HEADER_ONLY
+	#pragma once
+#endif
+
+// Internet.cpp : Defines the entry point for the DLL application.
+//
+#include "Internet.hpp"
+#include "../../http/hirose/httplib.h"
+
+#include <string.h>
+#include <stdlib.h>
+#include <string>
+
+#include <tjp/core/string/String.hpp>
+#include <tjp/core/algorithm/ExecuteOnDestruct.hpp>
+
+#include <tjp/core/future/Future.hpp>
+#include <tjp/core/log/Log.h>
+#include <tjp/core/log/LogOf.h>
+
+#include <tjp/core/string/as_vector.hpp>
+#include <tjp/core/string/as_string.hpp>
+
+//===============================================================================
+
+namespace tjp {
+namespace core {
+namespace internet {
+
+TJP_CORE_HEADER_ONLY_INLINE
+Future<FileContents> post (const std::string &url, Data &&data, const HttpOptions &options)
+{
+	Promise<FileContents> promise;
+	
+	post(url, std::move(data), options, promise);
+	
+	return promise.get_future();
+}
+
+TJP_CORE_HEADER_ONLY_INLINE
+void perform (const std::string &url, const std::string &method, Data &&data, const HttpOptions &options, Promise<FileContents> &promise)
+{
+	xLogDebug (logVar(url));
+
+	auto getFile = [url, options, method, data=std::move(data)]() {
+		sLogDebug("core::internet", "starting " << logVar(url));
+
+		FileContents contents;
+		
+		auto firstSlashSlash = url.find("//");
+		auto firstSlash = url.find("/", firstSlashSlash+2);
+		httplib::Client cli(url.substr(0, firstSlash));
+		if (options.timeout)
+			cli.set_read_timeout(options.timeout);
+		
+		httplib::Headers headers;
+		for (auto &[k,v]: options.headers)
+			headers.insert({k, v});
+
+		decltype(cli.Get(url.substr(firstSlash), headers)) result;
+		if (method == "GET")
+			result = cli.Get(url.substr(firstSlash), headers);
+		else
+		if (method == "POST")
+			result = cli.Post(url.substr(firstSlash), headers, data, "text/plain");
+		else
+		if (method == "PUT")
+			result = cli.Put(url.substr(firstSlash), headers, data, "application/binary");
+
+		if (options.onHeaders)
+			options.onHeaders(contents.headers, contents.statusCode);
+
+		if (options.onData)
+			options.onData(contents.data.data(), contents.data.size());
+			
+		if (options.onComplete)
+			options.onComplete(contents.statusCode);
+
+		if (result)
+		{
+			contents.statusCode = result->status;
+			contents.data = as_vector(result->body);
+			for (auto &header: result->headers)
+				contents.headers.push_back(header);
+			contents.lastModifiedTime = 0;
+		}
+		else
+		{
+			sLogRelease("errors", "get http failed " << logVar(url) << logVar(result.error()));
+			throw Exception (
+				"HTTPConnection getFile failed " + std::to_string((int)result.error()),
+				std::move(contents)
+			);
+		}
+		if (options.throwOnFailure && !contents.ok())
+		{
+			sLogRelease("errors", "get http failed " << logVar(url) << logVar(contents.statusCode));
+			throw Exception (
+				"HTTPConnection getFile failed code " + std::to_string(contents.statusCode),
+				std::move(contents)
+			);
+		}
+
+		return contents;
+	};
+	
+	promise_of(promise, getFile);
+}
+
+TJP_CORE_HEADER_ONLY_INLINE
+void get (const std::string &url,const HttpOptions &options, Promise<FileContents> &promise)
+{
+	return perform(url, "GET", Data {}, options, promise);
+}
+
+
+TJP_CORE_HEADER_ONLY_INLINE
+void post (const std::string &url, Data &&data, const HttpOptions &options, Promise<FileContents> &promise)
+{
+	return perform(url, "POST", std::move(data), options, promise);
+}
+
+TJP_CORE_HEADER_ONLY_INLINE
+void put (const std::string &url, Data &&data, const HttpOptions &options, Promise<FileContents> &promise)
+{
+	return perform(url, "PUT", std::move(data), options, promise);
+}
+
+TJP_CORE_HEADER_ONLY_INLINE
+void head (const std::string &url, Data &&data, const HttpOptions &options, Promise<FileContents> &promise)
+{
+	return perform(url, "HEAD", std::move(data), options, promise);
+}
+
+} // namespace
+} // namespace
+} // namespace

tjp/core/internet/hirose/Internet.hpp

@@ -0,0 +1,16 @@
+// TJP COPYRIGHT HEADER
+
+#pragma once
+
+#include "../Internet.hpp"
+
+namespace tjp::core::internet {
+
+void get (const std::string &url, const HttpOptions &options, Promise<FileContents> &promise);
+void post (const std::string &url, Data &&, const HttpOptions &options, Promise<FileContents> &promise);
+
+} // namespace
+
+#ifdef TJP_CORE_HEADER_ONLY
+	#include "Internet.cpp"
+#endif

tjp/core/internet/html_decode.hpp

@@ -0,0 +1,66 @@
+// TJP COPYRIGHT HEADER
+
+#pragma once
+
+#include <tjp/core/string/StringView.h>
+#include <tjp/core/io/memstream.hpp>
+
+#include <cctype>
+#include <iomanip>
+#include <sstream>
+#include <string>
+
+namespace tjp::core::internet {
+
+inline
+String html_decode(const StringView &value)
+{
+    std::ostringstream decoded;
+	imemstream input(value.data(), value.size());
+    char c;
+    
+    // Map of common HTML entities
+    static const std::unordered_map<std::string, char> html_entities = {
+        {"quot", '"'}, {"amp", '&'}, {"lt", '<'}, {"gt", '>'}, {"nbsp", ' '},
+        {"apos", '\''}
+    };
+
+    while (input.get(c)) {
+        if (c == '%') { // URL decoding
+            char hex[3] = {0};
+            if (input.get(hex[0]) && input.get(hex[1])) {
+                decoded << static_cast<char>(std::stoi(hex, nullptr, 16));
+            }
+        } else if (c == '+') {
+            decoded << ' ';
+        } else if (c == '&') { // HTML entity decoding
+            std::string entity;
+            char entity_char;
+            while (input.get(entity_char) && entity_char != ';' && entity.size() < 10) {
+                entity += entity_char;
+            }
+            
+            if (html_entities.count(entity)) {
+                decoded << html_entities.at(entity);
+            } else if (!entity.empty() && entity[0] == '#') {
+                try {
+                    if (entity[1] == 'x' || entity[1] == 'X') {
+                        decoded << static_cast<char>(std::stoi(entity.substr(2), nullptr, 16));
+                    } else {
+                        decoded << static_cast<char>(std::stoi(entity.substr(1), nullptr, 10));
+                    }
+                } catch (...) {
+                    decoded << "&" << entity << ";"; // Preserve invalid entity
+                }
+            } else {
+                decoded << "&" << entity << ";"; // Preserve unknown entity
+            }
+        } else {
+            decoded << c;
+        }
+    }
+    
+    return decoded.str();
+}
+
+} // namespace

tjp/core/internet/ios/Internet.hpp

@@ -0,0 +1,18 @@
+// TJP COPYRIGHT HEADER
+
+#include <tjp/core/header_only/compile.h>
+
+#include "../Internet.hpp"
+#include "../Internet+Internal.hpp"
+
+namespace tjp::core::internet {
+
+void perform (const WeakPtr<internal::Request> &request);
+
+} // namespace
+
+#ifdef TJP_CORE_HEADER_ONLY
+#ifdef __OBJC__
+	#include "Internet.mm"
+#endif
+#endif

tjp/core/internet/ios/Internet.mm

@@ -0,0 +1,614 @@
+#include <tjp/core/header_only/compile.h>
+
+#ifdef TJP_CORE_HEADER_ONLY
+	#pragma once
+#endif
+
+#pragma clang diagnostic ignored "-Wnullability-completeness"
+
+#import <Foundation/Foundation.h>
+
+#include "Internet.hpp"
+#include <string.h>
+#include <stdlib.h>
+#include <string>
+
+#include <tjp/core/future/Future.hpp>
+#include <tjp/core/log/Log.h>
+#include <tjp/core/log/LogOf.h>
+
+#include <tjp/core/string/as_string.hpp>
+
+// http://iphoneincubator.com/blog/server-communication/how-to-download-a-file-only-if-it-has-been-updated/comment-page-1
+
+//===============================================================================
+
+typedef bool (*PartialCallback)(const void *bytes, NSUInteger length, void * _Nonnull context);
+typedef void (*CompleteCallback)(int statusCode, const char * _Nullable errorMessage, void * _Nonnull context);
+typedef bool (*HeaderLineCallback)(const void *line, NSUInteger length, void * _Nonnull context);
+
+@interface ProgressiveDownloader : NSObject <NSURLSessionDataDelegate>
+{
+	PartialCallback partialCallback;
+	CompleteCallback completeCallback;
+	HeaderLineCallback headerLineCallback;
+	NSString *method;
+	
+	size_t maxRedirects;
+	void *callbackContext;
+}
+
+@property (nonatomic, strong) NSURLSession *session;
+@property (nonatomic, strong) NSMutableData *receivedData;
+
+- (instancetype)initWithPartial:(PartialCallback)partial
+					  complete:(CompleteCallback)complete
+						headers:(HeaderLineCallback)headers
+					   context:(void *)context
+					   maxRedirects:(size_t)maxRedirects
+					   method:(NSString *)method;
+
+- (void)startDownload:(NSMutableURLRequest *)url;
+
+- (void)URLSession:(NSURLSession *)session
+		  dataTask:(NSURLSessionDataTask *)dataTask
+	didReceiveData:(NSData *)data;
+
+@end
+
+@implementation ProgressiveDownloader
+
+- (instancetype)initWithPartial:(PartialCallback)partial
+					  complete:(CompleteCallback)complete
+					  headers:(HeaderLineCallback)headers
+					   context:(void *)context
+					   maxRedirects:(size_t)maxRedirects_
+					   method:(NSString *)method_
+{
+	self = [super init];
+	if (self)
+	{
+		partialCallback = partial;
+		completeCallback = complete;
+		headerLineCallback = headers;
+		callbackContext = context;
+		maxRedirects = maxRedirects_;
+		method = method_;
+
+		NSURLSessionConfiguration *config = [NSURLSessionConfiguration defaultSessionConfiguration];
+		config.HTTPCookieStorage = nil;
+		
+		NSURLSession *session = [NSURLSession sessionWithConfiguration:config delegate:self delegateQueue:[[NSOperationQueue alloc] init]];
+		self.session = session;
+		
+//		NSURLSessionConfiguration *config = [NSURLSessionConfiguration defaultSessionConfiguration];
+//		self.session = [NSURLSession sessionWithConfiguration:config
+//													 delegate:self
+//												delegateQueue:[NSOperationQueue mainQueue]];
+
+		self.receivedData = [NSMutableData data];
+	}
+	return self;
+}
+
+- (void)startDownload:(NSMutableURLRequest *)request
+{
+	NSURLSessionDataTask *task = [self.session dataTaskWithRequest:request];
+	[task resume];
+}
+
+// Response received
+- (void)URLSession:(NSURLSession *)session
+		  dataTask:(NSURLSessionDataTask *)dataTask
+didReceiveResponse:(NSURLResponse *)response
+ completionHandler:(void (^)(NSURLSessionResponseDisposition))completionHandler
+{
+	NSHTTPURLResponse *httpResponse = (NSHTTPURLResponse *)response;
+
+	if (httpResponse.statusCode/100 == 3)
+	if (maxRedirects)
+	{
+		sLogDebug("core::internet::headers", "getting headers while redirecting - ignoring");
+	
+		return completionHandler(NSURLSessionResponseAllow);
+	}
+	
+	auto continue_task = true;
+	if (headerLineCallback)
+	{
+		NSString *headerLine = [NSString stringWithFormat:@"HTTP/1.1 %d\r\n", httpResponse.statusCode];
+		headerLineCallback(headerLine.UTF8String, [headerLine length], callbackContext);
+
+		[httpResponse.allHeaderFields enumerateKeysAndObjectsUsingBlock:^(id key, id value, BOOL *stop) {
+			NSString *k = [key description];
+			NSString *v = [value description];
+			NSString *headerLine = [NSString stringWithFormat:@"%@: %@\r\n", k, v];
+			headerLineCallback(headerLine.UTF8String, [headerLine length], callbackContext);
+		}];
+
+		continue_task = headerLineCallback("\r\n", 2, callbackContext);
+	}
+
+	if (!continue_task)
+		[dataTask cancel];
+		
+	completionHandler(NSURLSessionResponseAllow);
+}
+
+// Data arrives progressively
+- (void)URLSession:(NSURLSession *)session
+		  dataTask:(NSURLSessionDataTask *)dataTask
+	didReceiveData:(NSData *)data
+{
+	auto continue_task = true;
+	if (partialCallback)
+		continue_task = partialCallback(data.bytes, data.length, callbackContext);
+
+	[self.receivedData appendData:data];
+	
+	if (!continue_task)
+		[dataTask cancel];
+}
+
+// Completion or error
+- (void)URLSession:(NSURLSession * _Nonnull)session
+			  task:(NSURLSessionTask * _Nonnull)task
+didCompleteWithError:(NSError * _Nullable)error
+{
+	NSHTTPURLResponse *response = (NSHTTPURLResponse *)task.response;
+	
+	if (completeCallback)
+	{
+		if (error)
+			completeCallback(response.statusCode, error.localizedDescription.UTF8String, callbackContext);
+		else
+			completeCallback(response.statusCode, NULL, callbackContext);
+	}
+}
+
+// Optionally handle redirect logic if needed
+- (void)URLSession:(NSURLSession * _Nonnull)session
+        task:(NSURLSessionTask * _Nonnull)task
+          willPerformHTTPRedirection:(NSHTTPURLResponse *)response_
+          newRequest:(NSURLRequest *)request
+          completionHandler:(void (^)(NSURLRequest * _Nullable))completionHandler
+{
+//	NSHTTPURLResponse *response = (NSHTTPURLResponse *)response_;
+
+	sLogDebug("core::internet", "redirect requested " <<
+		logVar(maxRedirects)
+		logVar(request.URL.absoluteString.UTF8String) <<
+		logVar(request.HTTPMethod.UTF8String)
+	);
+
+	if (maxRedirects)
+	{
+		maxRedirects--;
+		
+		if (![request.HTTPMethod isEqualToString:method])
+		{
+			sLogDebug("core::internet", "modifying method to match request");
+		
+			NSMutableURLRequest *request_ = request.mutableCopy;
+			[request_ setHTTPMethod:method];
+			request = request_;
+		}
+		
+		completionHandler(request);
+	}
+	else
+	{
+		completionHandler(nil);
+	}
+}
+
+@end
+
+
+namespace tjp {
+namespace core {
+namespace internet {
+
+struct ProgressiveContext {
+	internal::ProgressiveData data;
+	dispatch_semaphore_t semaphor;
+} ;
+
+bool onPartial(const void * _Nullable bytes, NSUInteger length, void * _Nonnull context_)
+{
+	return internal::receiveContentsCallback((char *)bytes, length, context_) == length;
+}
+
+bool onHeaders(const void * _Nullable bytes, NSUInteger length, void * _Nonnull context_)
+{
+	return internal::receiveHeadersCallback((char *)bytes, length, context_) == length;
+}
+
+void onComplete(int statusCode, const char * _Nullable errorMessage, void * _Nonnull context_)
+{
+	auto *context = static_cast<ProgressiveContext *>(context_);
+	context->data.receive.data.statusCode = statusCode;
+		
+	dispatch_semaphore_signal(context->semaphor);
+}
+
+Future<FileContents> get (const std::string &url, const HttpOptions &options)
+{
+	auto request = strong<internal::Request>();
+	request->type = internal::RequestType::GET;
+	request->url = url;
+	request->options = options;
+	
+	perform(request);
+	
+	return request->promise.get_future();
+}
+
+Future<FileContents> post (const std::string &url, Data &&data, const HttpOptions &options)
+{
+	auto request = strong<internal::Request>();
+	request->type = internal::RequestType::POST;
+	request->url = url;
+	request->data = std::move(data);
+	request->options = options;
+	
+	perform(request);
+	
+	return request->promise.get_future();
+}
+
+
+void perform_progressive(const WeakPtr<internal::Request> &request_)
+{
+	auto getFile = [](const WeakPtr<internal::Request> &request_)
+	{
+		ProgressiveContext context;
+		auto &data = context.data;
+		data.request = request_;
+		context.semaphor = dispatch_semaphore_create(0);
+		
+		String url;
+		ProgressiveDownloader *downloader;
+		NSString *method;
+
+		if (auto request__ = strong(request_))
+		{
+			url = request__->url;
+			
+			data.options = std::move(request__->options);
+			if (request__->data)
+				data.send.data = std::move(*request__->data);
+
+			NSString *stringURL = [NSString stringWithUTF8String:url.c_str()];
+			
+			// without this, iOS <= 16 doesn't work for all urls
+			NSCharacterSet *allowed = [NSCharacterSet URLFragmentAllowedCharacterSet];
+			NSString *stringURLEncoded = [stringURL stringByAddingPercentEncodingWithAllowedCharacters:allowed];
+			
+			NSURL *url_ = [NSURL URLWithString:stringURLEncoded];
+
+			sLogDebug("core::internet", "starting " << logVar(url) << logVar(stringURLEncoded.UTF8String));
+
+			// Prepare HTTP request
+			NSMutableURLRequest *request = [[NSMutableURLRequest alloc] initWithURL:url_];
+			[request setCachePolicy:NSURLRequestReloadIgnoringCacheData];
+	//		[request setValue:@"gzip" forHTTPHeaderField:@"Accept-Encoding"];
+
+			if (data.options.timeout)
+				request.timeoutInterval = data.options.timeout;
+
+			auto &requestType = request__->type;
+			if (requestType == internal::RequestType::POST)
+			{
+				method = @"POST";
+
+				sLogDebug("core::internet", "starting " << "POST");
+				[request setHTTPMethod:method];
+				NSData *data_ = [NSData dataWithBytes:data.send.data.c_str() length:data.send.data.size()];
+				[request setHTTPBody:data_];
+			}
+			else
+			if (requestType == internal::RequestType::PUT)
+			{
+				method = @"PUT";
+
+				sLogDebug("core::internet", "starting " << "PUT");
+				[request setHTTPMethod:method];
+				NSData *data_ = [NSData dataWithBytes:data.send.data.data() length:data.send.data.size()];
+				[request setHTTPBody:data_];
+			}
+			else
+			if (requestType == internal::RequestType::GET)
+			{
+				method = @"GET";
+
+				sLogDebug("core::internet", "starting " << "GET");
+				[request setHTTPMethod:method];
+			}
+			else
+			if (requestType == internal::RequestType::HEAD)
+			{
+				method = @"HEAD";
+
+				sLogDebug("core::internet", "starting " << "HEAD");
+				[request setHTTPMethod:method];
+			}
+
+			for (auto &header : data.options.headers)
+			{
+				auto &[key, value] = header;
+				sLogDebug("core::internet::headers", "send header " << logVar(key) << logVar(value));
+
+				NSString *key_ = [NSString stringWithUTF8String:key.c_str()];
+				NSString *value_ = [NSString stringWithUTF8String:value.c_str()];
+				[request setValue:value_ forHTTPHeaderField:key_];
+			}
+
+			if (data.options.lastModifiedTime)
+			{
+				NSDate *date = [NSDate dateWithTimeIntervalSince1970:data.options.lastModifiedTime];
+				NSDateFormatter *df = [[NSDateFormatter alloc] init];
+				df.dateFormat = @"EEE, dd MMM yyyy HH:mm:ss Z";
+				df.locale = [[NSLocale alloc] initWithLocaleIdentifier:@"en_US_POSIX"];
+				df.timeZone = [NSTimeZone timeZoneWithAbbreviation:@"GMT"];
+				NSString *dateString = [df stringFromDate:date];
+				[request setValue:dateString forHTTPHeaderField:@"If-Modified-Since"];
+			}
+			
+			if (url_.user)
+			{
+				NSString *user = url_.user;
+				NSString *pass = url_.password ?: @"";
+				NSString *auth = [NSString stringWithFormat:@"%@:%@", user, pass];
+
+				NSData *authData = [auth dataUsingEncoding:NSUTF8StringEncoding];
+				NSString *authValue = [NSString stringWithFormat:@"Basic %@",
+					[authData base64EncodedStringWithOptions:0]];
+
+				[request setValue:authValue forHTTPHeaderField:@"Authorization"];
+			}
+
+			[request setValue:@"close" forHTTPHeaderField:@"Connection"];
+
+			// --- Define progressive callbacks ---
+
+			downloader =
+				[[ProgressiveDownloader alloc]
+					initWithPartial:onPartial
+					complete:onComplete
+					headers:onHeaders
+					context:&context
+					maxRedirects:data.options.maxRedirects
+					method:method
+			];
+
+			[downloader startDownload:request];
+		}
+		else
+		{
+			throw Exception (
+				"HTTPConnection canceled",
+				{}
+			);
+		}
+
+		dispatch_semaphore_wait(context.semaphor, DISPATCH_TIME_FOREVER);
+
+		sLogDebug("core::internet", "finishing 0");
+
+		if (data.receive.data.statusCode < 0)
+			data.receive.data.statusCode = internet::GeneralFailure;
+
+		if (data.interrupted)
+			data.receive.data.statusCode = internet::GeneralFailure;
+
+		if (data.options.onHeaders)
+			data.options.onHeaders(data.receive.data.headers, data.receive.data.statusCode);
+		
+		if (data.options.onComplete)
+			data.options.onComplete(data.receive.data.statusCode);
+
+		sLogDebug("core::internet", "finishing 1");
+
+		auto got_error = data.receive.data.statusCode >= 400;
+
+		sLogReleaseIf(got_error, "debug", "finished " << logVar(url) << logVar(got_error));
+
+		if (got_error)
+		{
+			throw Exception("HTTPConnection getFile failed", std::move(data.receive.data));
+		}
+
+		if (data.options.throwOnFailure && !data.receive.data.ok())
+		{
+			sLogRelease("errors", "get http failed: " << logVar(url) << logVar(data.receive.data.statusCode));
+			throw Exception(
+				"HTTPConnection getFile failed code " + std::to_string(data.receive.data.statusCode),
+				std::move(data.receive.data));
+		}
+
+		return data.receive.data;
+	};
+
+	Promise<FileContents> promise;
+	if (auto request = strong(request_))
+		promise = std::move(request->promise);
+
+	promise.consume(getFile, request_);
+}
+
+#if REMOVE_CODE_2025_11_11
+# code is superseded by perform
+
+void perform_completion (const std::string &url, const std::string &method, Data &&data, const HttpOptions &options, Promise<FileContents> &promise)
+{
+	xLogDebug (logVar(url));
+
+	auto getFile = [&]() {
+		__block FileContents contents;
+		
+		sLogDebug("core::internet", "starting " << url);
+		
+		NSString *stringURL = [NSString stringWithUTF8String:url.c_str()];
+		NSURL  *url_ = [NSURL URLWithString:stringURL];
+
+//		NSURLSessionConfiguration *config = [NSURLSessionConfiguration ephemeralSessionConfiguration]; // No cookie storage
+		NSURLSessionConfiguration *config = [NSURLSessionConfiguration defaultSessionConfiguration]; // No cookie storage
+//		config.HTTPCookieAcceptPolicy = NSHTTPCookieAcceptPolicyNever;
+		config.HTTPCookieStorage = nil;
+		
+		NSURLSession *session = [NSURLSession sessionWithConfiguration:config];
+//		NSURLSession *session = [NSURLSession sharedSession];
+		NSMutableURLRequest *request = [[NSMutableURLRequest alloc] initWithURL:url_];
+		
+		if (options.timeout)
+			request.timeoutInterval = options.timeout;
+
+		[request setValue:@"gzip" forHTTPHeaderField:@"Accept-Encoding"];
+		[request setCachePolicy:NSURLRequestReloadIgnoringCacheData];
+		
+		if (method == "POST")
+		{
+			[request setHTTPMethod:@"POST"];
+			
+			NSString *str = [NSString stringWithUTF8String:data.c_str()];
+			NSData *data_ = [str dataUsingEncoding:NSUTF8StringEncoding];
+
+			[request setHTTPBody:data_];
+		}
+		else
+		if (method == "GET")
+		{
+			[request setHTTPMethod:@"GET"];
+		}
+		else
+		if (method == "HEAD")
+		{
+			[request setHTTPMethod:@"HEAD"];
+		}
+		else
+		if (method == "PUT")
+		{
+			[request setHTTPMethod:@"PUT"];
+			
+			NSData *data_ = [NSData dataWithBytes:data.data() length:data.size()];
+			[request setHTTPBody:data_];
+		}
+
+		for (auto &header : options.headers)
+		{
+			auto &[key, value] = header;
+			NSString *key_ = [NSString stringWithUTF8String:key.c_str()];
+			NSString *value_ = [NSString stringWithUTF8String:value.c_str()];
+			
+			[request setValue:value_ forHTTPHeaderField:key_];
+		}
+		
+		if (options.lastModifiedTime)
+		{
+			NSString *key_ = @"If-Modified-Since";
+			NSDate *date = [NSDate dateWithTimeIntervalSince1970:options.lastModifiedTime];
+			NSDateFormatter *dateFormatter = [[NSDateFormatter alloc] init]; // maybe there exist a new-method now
+			dateFormatter.dateFormat = @"EEE, dd MMM yyyy HH:mm:ss Z"; //RFC2822-Format
+			dateFormatter.locale = [[NSLocale alloc] initWithLocaleIdentifier: @"en_US_POSIX"];
+			dateFormatter.timeZone = [NSTimeZone timeZoneWithAbbreviation:@"GMT"];
+			NSString *dateString = [dateFormatter stringFromDate:date];
+
+			[request setValue:dateString forHTTPHeaderField:key_];
+		}
+
+		dispatch_semaphore_t semaphore = dispatch_semaphore_create(0);
+		__block bool got_error = false;
+
+		NSURLSessionDataTask *downloadTask =
+			[session dataTaskWithRequest:request completionHandler:^(NSData *data, NSURLResponse *response_, NSError *error) {
+				if (!error)
+				{
+					NSHTTPURLResponse *response = (NSHTTPURLResponse *)response_;
+					NSString *lastModifiedString = [[response allHeaderFields] objectForKey:@"Last-Modified"];
+
+					if (lastModifiedString)
+					{
+						NSDate *lastModifiedServer = nil;
+						@try {
+							NSDateFormatter *df = [[NSDateFormatter alloc] init];
+							df.dateFormat = @"EEE',' dd MMM yyyy HH':'mm':'ss 'GMT'";
+							df.locale = [[NSLocale alloc] initWithLocaleIdentifier:@"en_US"];
+							df.timeZone = [NSTimeZone timeZoneWithAbbreviation:@"GMT"];
+							lastModifiedServer = [df dateFromString:lastModifiedString];
+						}
+						@catch (NSException * e) {
+							NSLog(@"Error parsing last modified date: %@ - %@", lastModifiedString, [e description]);
+						}
+						 
+						contents.lastModifiedTime = [lastModifiedServer timeIntervalSince1970];
+					}
+					
+					NSDictionary *headers = [response allHeaderFields];
+					NSArray *keys = [headers allKeys];
+					
+					for (NSString* key in keys)
+					{
+						NSString *value = [headers objectForKey:key];
+						contents.headers.push_back({ key.UTF8String, value.UTF8String });
+					}
+					
+					contents.data.resize(data.length);
+					contents.statusCode = response.statusCode;
+					memcpy(contents.data.data(), data.bytes, contents.data.size());
+				}
+				else
+				{
+					got_error = true;
+				}
+				
+				dispatch_semaphore_signal(semaphore);
+			}];
+
+		[downloadTask resume];
+
+		dispatch_semaphore_wait(semaphore, DISPATCH_TIME_FOREVER);
+
+		if (options.onHeaders)
+			options.onHeaders(contents.headers, contents.statusCode);
+		
+		if (options.onData)
+			options.onData(contents.data.data(), contents.data.size());
+			
+		if (options.onComplete)
+			options.onComplete(contents.statusCode);
+
+		sLogReleaseIf(got_error, "debug", "finished " << logVar(url) << logVar(got_error));
+
+		if (got_error)
+			throw Exception (
+				"HTTPConnection getFile failed",
+				std::move(contents)
+			);
+
+		if (options.throwOnFailure && !contents.ok())
+		{
+			sLogRelease("errors", "get http failed: " << logVar(url) << logVar(contents.statusCode));
+			throw Exception (
+				"HTTPConnection getFile failed code " + std::to_string(contents.statusCode),
+				std::move(contents)
+			);
+		}
+
+		return contents;
+	};
+	
+	promise_of(promise, getFile);
+}
+
+#endif
+
+void perform(const WeakPtr<internal::Request> &request_)
+{
+	perform_progressive(request_);
+}
+
+
+} // namespace
+} // namespace
+} // namespace

tjp/core/internet/ios/Queue.cpp

@@ -0,0 +1,106 @@
+// TJP COPYRIGHT HEADER
+
+#ifdef TJP_CORE_HEADER_ONLY
+	#pragma once
+#endif
+
+#include "../Queue.hpp"
+#include "Internet.hpp"
+
+#include <tjp/core/threads/Lock.hpp>
+#include <tjp/core/threads/process_queue.hpp>
+
+#include <tjp/core/ptr/strong_of.hpp>
+#include <tjp/core/future/Future.hpp>
+
+#include <queue>
+
+//===============================================================================
+
+namespace tjp::core::internet {
+
+struct Queue::Internal
+{
+	std::thread thread;
+	bool finished = false;
+
+	std::queue<WeakPtr<internal::Request>> queue;
+	Event event;
+	Event::Mutex mutex;
+	
+	void run ();
+	void runOne (const WeakPtr<internal::Request> &i);
+} ;
+
+TJP_CORE_HEADER_ONLY_INLINE
+Queue::Queue()
+{
+}
+
+TJP_CORE_HEADER_ONLY_INLINE
+Queue::~Queue ()
+{
+}
+
+TJP_CORE_HEADER_ONLY_INLINE
+void Queue::Internal::runOne (const WeakPtr<internal::Request> &i_)
+{
+	perform(i_);
+}
+
+TJP_CORE_HEADER_ONLY_INLINE
+void Queue::Internal::run ()
+{
+	threads::process_queue(
+		finished,
+		event,
+		mutex,
+		queue,
+		[this](auto &i) { return runOne(i); }
+	);
+}
+
+TJP_CORE_HEADER_ONLY_INLINE
+void Queue::prepare()
+{
+	if (i)
+		return;
+		
+	static Mutex mutex;
+	auto lock = lock_of(mutex);
+	if (i)
+		return;
+		
+	i = strong<Internal>();
+	i->thread = std::thread([s=i](){ s->run(); });
+}
+
+TJP_CORE_HEADER_ONLY_INLINE
+Future<FileContents> Queue::execute(internal::RequestType type, const std::string &url, Data &&data, const HttpOptions &options)
+{
+	prepare();
+
+	auto item = strong_of_value(
+		internal::Request {
+			.type = type,
+			.url = url,
+			.options = options,
+			.data = std::move(data)
+		}
+	);
+		
+	{
+		auto lock = lock_of(i->mutex);
+		i->queue.push(weak(item));
+	}
+	
+	i->event.notify_all();
+	
+	return item->promise.get_future().then(
+		[item](auto &&f) {
+			return f.get();
+		}
+	);
+}
+
+} // namespace

tjp/core/internet/ios/Queue.hpp

@@ -0,0 +1,7 @@
+// TJP COPYRIGHT HEADER
+
+#pragma once
+
+#ifdef TJP_CORE_HEADER_ONLY
+	#include "Queue.cpp"
+#endif

tjp/core/internet/join.hpp

@@ -0,0 +1,39 @@
+// TJP COPYRIGHT HEADER
+
+#pragma once
+
+#include <tjp/core/string/String.h>
+#include <tjp/core/string/StringView+concat.hpp>
+
+namespace tjp::core::internet {
+
+inline
+void join_(String &base, const StringView &url)
+{
+	if (base.empty())
+	{
+		concat_(base, url);
+	}
+	else
+	if (!url.empty())
+	{
+		if (base.back() == '/' && url.front() == '/')
+			concat_(base, url.substr(1));
+		else
+		if (base.back() != '/' && url.front() != '/')
+			concat_(base, "/", url);
+		else
+			concat_(base, url);
+	}
+}
+
+template<typename ... Args>
+String join(Args&& ...args)
+{
+	String result;
+	(join_(result, std::forward<Args>(args)), ...);
+	
+	return result;
+}
+
+} // namespace

tjp/core/internet/url_param_decode.hpp

@@ -0,0 +1,39 @@
+// TJP COPYRIGHT HEADER
+
+#pragma once
+
+#include <tjp/core/string/String.h>
+#include <tjp/core/string/StringView+concat.hpp>
+#include <tjp/core/io/memstream.hpp>
+
+#include <cctype>
+#include <iomanip>
+#include <sstream>
+#include <string>
+
+namespace tjp::core::internet {
+
+inline
+String url_param_decode(const StringView &value)
+{
+	std::ostringstream decoded;
+	io::imemstream input(value.data(), value.size());
+	char c;
+
+	while (input.get(c)) {
+		if (c == '%') {
+			char hex[3] = {0};
+			if (input.get(hex[0]) && input.get(hex[1])) {
+				decoded << static_cast<char>(std::stoi(hex, nullptr, 16));
+			}
+		} else if (c == '+') {
+			decoded << ' ';
+		} else {
+			decoded << c;
+		}
+	}
+
+	return decoded.str();
+}
+
+} // namespace

tjp/core/internet/url_param_encode.hpp

@@ -0,0 +1,40 @@
+// TJP COPYRIGHT HEADER
+
+#pragma once
+
+#include <tjp/core/string/String.h>
+#include <tjp/core/string/StringView+concat.hpp>
+
+#include <cctype>
+#include <iomanip>
+#include <sstream>
+#include <string>
+
+namespace tjp::core::internet {
+
+inline
+String url_param_encode(const StringView &value)
+{
+	std::ostringstream escaped;
+	escaped.fill('0');
+	escaped << std::hex;
+
+	for (auto i = value.begin(), n = value.end(); i != n; ++i) {
+		auto c = (*i);
+
+		// Keep alphanumeric and other accepted characters intact
+		if (isalnum(c) || c == '-' || c == '_' || c == '.' || c == '~') {
+			escaped << c;
+			continue;
+		}
+
+		// Any other characters are percent-encoded
+		escaped << std::uppercase;
+		escaped << '%' << std::setw(2) << int((unsigned char) c);
+		escaped << std::nouppercase;
+	}
+
+	return escaped.str();
+}
+
+} // namespace

tjp/core/internet/url_path_decode.hpp

@@ -0,0 +1,15 @@
+// TJP COPYRIGHT HEADER
+
+#pragma once
+
+#include "url_param_decode.hpp"
+
+namespace tjp::core::internet {
+
+inline
+String url_path_decode(const StringView &value)
+{
+	return url_param_decode(value);
+}
+
+} // namespace

tjp/core/internet/url_path_encode.hpp

@@ -0,0 +1,40 @@
+// TJP COPYRIGHT HEADER
+
+#pragma once
+
+#include <tjp/core/string/String.h>
+#include <tjp/core/string/StringView+concat.hpp>
+
+#include <cctype>
+#include <iomanip>
+#include <sstream>
+#include <string>
+
+namespace tjp::core::internet {
+
+inline
+String url_path_encode(const StringView &value)
+{
+	std::ostringstream escaped;
+	escaped.fill('0');
+	escaped << std::hex;
+
+	for (auto i = value.begin(), n = value.end(); i != n; ++i) {
+		auto c = (*i);
+
+		// Keep alphanumeric and other accepted characters intact
+		if (isalnum(c) || c == ':' || c == '/' || c == '-' || c == '_' || c == '.' || c == '~') {
+			escaped << c;
+			continue;
+		}
+
+		// Any other characters are percent-encoded
+		escaped << std::uppercase;
+		escaped << '%' << std::setw(2) << int((unsigned char) c);
+		escaped << std::nouppercase;
+	}
+
+	return escaped.str();
+}
+
+} // namespace

tjp/core/internet/url_with_params.hpp

@@ -0,0 +1,16 @@
+// TJP COPYRIGHT HEADER
+
+#pragma once
+
+#include "form_encode.hpp"
+#include "url_path_encode.hpp"
+
+namespace tjp::core::internet {
+
+template<typename ... Args>
+String url_with_params(const StringView &url, Args && ...args)
+{
+	return url_path_encode(url) + "?" + form_encode(std::forward<Args>(args)...);
+}
+
+} // namespace