flatten 20260225
This commit is contained in:
Timothy Prepscius
289
tjp/core/internet/curl/Internet.cpp
Executable file
289
tjp/core/internet/curl/Internet.cpp
Executable file
@@ -0,0 +1,289 @@
|
||||
// TJP COPYRIGHT HEADER
|
||||
|
||||
#include <tjp/core/header_only/compile.h>
|
||||
|
||||
#ifdef TJP_CORE_HEADER_ONLY
|
||||
#pragma once
|
||||
#endif
|
||||
|
||||
#include "Internet.hpp"
|
||||
|
||||
#include <curl/curl.h>
|
||||
#include <string.h>
|
||||
#include <stdlib.h>
|
||||
#include <string>
|
||||
#include <cstdlib>
|
||||
|
||||
#include <tjp/core/string/String.hpp>
|
||||
#include <tjp/core/string/as_string_view.hpp>
|
||||
#include <tjp/core/algorithm/ExecuteOnDestruct.hpp>
|
||||
|
||||
#include <tjp/core/future/Future.hpp>
|
||||
#include <tjp/core/log/Log.h>
|
||||
#include <tjp/core/log/LogOf.h>
|
||||
|
||||
|
||||
//===============================================================================
|
||||
|
||||
namespace tjp {
|
||||
namespace core {
|
||||
namespace internet {
|
||||
|
||||
TJP_CORE_HEADER_ONLY_INLINE
|
||||
size_t receiveContentsCallback(void *ptr, size_t size, size_t nmemb, void *userdata)
|
||||
{
|
||||
return internal::receiveContentsCallback(ptr, size*nmemb, userdata);
|
||||
}
|
||||
|
||||
TJP_CORE_HEADER_ONLY_INLINE
|
||||
size_t sendContentsCallback(char *ptr, size_t size, size_t nmemb, void *userdata)
|
||||
{
|
||||
return internal::sendContentsCallback(ptr, size*nmemb, userdata);
|
||||
}
|
||||
|
||||
TJP_CORE_HEADER_ONLY_INLINE
|
||||
size_t receiveHeadersCallback(char *ptr, size_t size, size_t nitems, void *userdata)
|
||||
{
|
||||
return internal::receiveHeadersCallback(ptr, size*nitems, userdata);
|
||||
|
||||
}
|
||||
|
||||
TJP_CORE_HEADER_ONLY_INLINE
|
||||
Future<FileContents> get (const std::string &url, const HttpOptions &options)
|
||||
{
|
||||
auto *connection = curl_easy_init();
|
||||
|
||||
ExecuteOnDestruct cleanup = [&]() {
|
||||
curl_easy_cleanup(connection);
|
||||
};
|
||||
|
||||
auto request = strong<internal::Request>();
|
||||
request->type = internal::RequestType::GET;
|
||||
request->url = url;
|
||||
request->options = options;
|
||||
|
||||
perform(request, connection);
|
||||
|
||||
return request->promise.get_future();
|
||||
}
|
||||
|
||||
TJP_CORE_HEADER_ONLY_INLINE
|
||||
Future<FileContents> post (const std::string &url, Data &&data, const HttpOptions &options)
|
||||
{
|
||||
auto *connection = curl_easy_init();
|
||||
|
||||
ExecuteOnDestruct cleanup = [&]() {
|
||||
curl_easy_cleanup(connection);
|
||||
};
|
||||
|
||||
auto request = strong<internal::Request>();
|
||||
request->type = internal::RequestType::POST;
|
||||
request->url = url;
|
||||
request->options = options;
|
||||
request->data = std::move(data);
|
||||
|
||||
perform(request, connection);
|
||||
|
||||
return request->promise.get_future();
|
||||
}
|
||||
|
||||
TJP_CORE_HEADER_ONLY_INLINE
|
||||
void perform (const WeakPtr<internal::Request> &request_, CURL *connection)
|
||||
{
|
||||
sLogDebug("core::internet", "perform");
|
||||
|
||||
curl_easy_reset(connection);
|
||||
|
||||
auto getFile = [](const WeakPtr<internal::Request> &request_, CURL *connection) {
|
||||
|
||||
internal::ProgressiveData data;
|
||||
data.request = request_;
|
||||
String url;
|
||||
|
||||
struct curl_slist* headers = NULL;
|
||||
|
||||
if (auto request = strong(request_))
|
||||
{
|
||||
url = request->url;
|
||||
data.options = std::move(request->options);
|
||||
if (request->data)
|
||||
data.send.data = std::move(*request->data);
|
||||
|
||||
sLogDebug("core::internet", "starting " << logVar(url));
|
||||
|
||||
FileContents contents;
|
||||
|
||||
if (!connection)
|
||||
return contents;
|
||||
|
||||
CURLU *u = curl_url();
|
||||
char *user_ = nullptr;
|
||||
char *password_ = nullptr;
|
||||
Optional<String> user, password;
|
||||
|
||||
if (curl_url_set(u, CURLUPART_URL, url.c_str(), 0) == CURLUE_OK)
|
||||
{
|
||||
if (curl_url_get(u, CURLUPART_USER, &user_, 0) == CURLUE_OK)
|
||||
user = user_;
|
||||
|
||||
if (curl_url_get(u, CURLUPART_PASSWORD, &password_, 0) == CURLUE_OK)
|
||||
password = password_;
|
||||
|
||||
}
|
||||
curl_url_cleanup(u);
|
||||
|
||||
for (auto &header: data.options.headers)
|
||||
{
|
||||
auto &[key, value] = header;
|
||||
sLogDebug("core::internet::headers", "send header " << logVar(key) << logVar(value));
|
||||
|
||||
headers = curl_slist_append(headers, (key + ": " + value).c_str());
|
||||
}
|
||||
|
||||
curl_easy_setopt (connection, CURLOPT_HTTPHEADER, headers);
|
||||
|
||||
curl_easy_setopt(connection, CURLOPT_SSL_VERIFYPEER, false);
|
||||
|
||||
// if (CA_BUNDLE_PATH)
|
||||
// curl_easy_setopt(connection, CURLOPT_CAINFO, CA_BUNDLE_PATH);
|
||||
|
||||
curl_easy_setopt (connection, CURLOPT_USERAGENT, "User-Agent: Curl Client");
|
||||
// curl_easy_setopt (connection, CURLOPT_ACCEPT_ENCODING, "gzip, deflate");
|
||||
// curl_easy_setopt (connection, CURLOPT_ACCEPT_ENCODING, "");
|
||||
curl_easy_setopt (connection, CURLOPT_URL, url.c_str());
|
||||
curl_easy_setopt (connection, CURLOPT_WRITEFUNCTION, receiveContentsCallback);
|
||||
curl_easy_setopt (connection, CURLOPT_WRITEDATA, (void *)&data);
|
||||
curl_easy_setopt (connection, CURLOPT_HEADERFUNCTION, receiveHeadersCallback);
|
||||
curl_easy_setopt (connection, CURLOPT_HEADERDATA, (void *)&data);
|
||||
|
||||
if (user)
|
||||
curl_easy_setopt(connection, CURLOPT_USERNAME, user->c_str());
|
||||
if (password)
|
||||
curl_easy_setopt(connection, CURLOPT_PASSWORD, password->c_str());
|
||||
|
||||
curl_easy_setopt(connection, CURLOPT_HTTPAUTH, CURLAUTH_BASIC);
|
||||
|
||||
if (data.options.httpVersion == "2")
|
||||
curl_easy_setopt(connection, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
|
||||
|
||||
if (data.options.maxRedirects)
|
||||
{
|
||||
curl_easy_setopt(connection, CURLOPT_FOLLOWLOCATION, 1L);
|
||||
curl_easy_setopt(connection, CURLOPT_MAXREDIRS, data.options.maxRedirects);
|
||||
curl_easy_setopt(connection, CURLOPT_POSTREDIR, CURL_REDIR_POST_ALL);
|
||||
}
|
||||
|
||||
if (request->type == internal::RequestType::GET)
|
||||
{
|
||||
curl_easy_setopt(connection, CURLOPT_HTTPGET, 1L);
|
||||
}
|
||||
else
|
||||
if (request->type == internal::RequestType::HEAD)
|
||||
{
|
||||
curl_easy_setopt(connection, CURLOPT_NOBODY, 1);
|
||||
}
|
||||
else
|
||||
if (request->type == internal::RequestType::POST)
|
||||
{
|
||||
DataView block = data.send.data;
|
||||
curl_easy_setopt(connection, CURLOPT_POST, 1);
|
||||
curl_easy_setopt(connection, CURLOPT_POSTFIELDS, block.data());
|
||||
curl_easy_setopt(connection, CURLOPT_POSTFIELDSIZE_LARGE, block.size());
|
||||
}
|
||||
else
|
||||
if (request->type == internal::RequestType::PUT)
|
||||
{
|
||||
curl_easy_setopt(connection, CURLOPT_UPLOAD, 1);
|
||||
curl_easy_setopt(connection, CURLOPT_READDATA, &data);
|
||||
curl_easy_setopt(connection, CURLOPT_READFUNCTION, sendContentsCallback);
|
||||
curl_easy_setopt(connection, CURLOPT_INFILESIZE_LARGE, (curl_off_t)data.send.data.size());
|
||||
}
|
||||
|
||||
if (data.options.timeout)
|
||||
curl_easy_setopt(connection, CURLOPT_TIMEOUT, data.options.timeout);
|
||||
|
||||
// curl_easy_setopt (connection, CURLOPT_DNS_CACHE_TIMEOUT, 0L);
|
||||
// curl_easy_setopt (connection, CURLOPT_NOSIGNAL, 1L);
|
||||
// curl_easy_setopt (connection, CURLOPT_DNS_USE_GLOBAL_CACHE, 0L);
|
||||
curl_easy_setopt (connection, CURLOPT_FILETIME, 1L);
|
||||
|
||||
if (data.options.verbose)
|
||||
curl_easy_setopt(connection, CURLOPT_VERBOSE, 1L);
|
||||
|
||||
}
|
||||
else
|
||||
{
|
||||
throw Exception (
|
||||
"HTTPConnection canceled",
|
||||
{}
|
||||
);
|
||||
}
|
||||
|
||||
StatusCode code;
|
||||
CURLcode res = curl_easy_perform(connection);
|
||||
curl_slist_free_all(headers);
|
||||
|
||||
if (res != CURLE_OK)
|
||||
{
|
||||
data.receive.data.statusCode = GeneralFailure;
|
||||
}
|
||||
else
|
||||
{
|
||||
curl_easy_getinfo (connection, CURLINFO_RESPONSE_CODE, &code);
|
||||
data.receive.data.statusCode = code;
|
||||
xLogDebug (logVar(code));
|
||||
}
|
||||
|
||||
StatusCode lastModifiedTime;
|
||||
curl_easy_getinfo(connection, CURLINFO_FILETIME, &lastModifiedTime);
|
||||
data.receive.data.lastModifiedTime = lastModifiedTime;
|
||||
|
||||
if (data.receive.data.statusCode < 0)
|
||||
data.receive.data.statusCode = GeneralFailure-1;
|
||||
|
||||
if (data.interrupted)
|
||||
data.receive.data.statusCode = GeneralFailure-2;
|
||||
|
||||
sLogDebugIf(res, "debug", "finishing " << logVar(url) << logVar(res) << logVar(data.receive.data.statusCode));
|
||||
|
||||
if (data.options.onHeaders)
|
||||
data.options.onHeaders(data.receive.data.headers, (int)data.receive.data.statusCode);
|
||||
|
||||
if (data.options.onComplete)
|
||||
data.options.onComplete(data.receive.data.statusCode);
|
||||
|
||||
sLogDebugIf(res, "debug", "finished " << logVar(url) << logVar(res) << logVar(data.receive.data.statusCode));
|
||||
|
||||
if (res > 0)
|
||||
{
|
||||
sLogRelease("errors", "get http failed " << logVar(url) << logVar(res));
|
||||
throw Exception (
|
||||
"HTTPConnection getFile failed " + std::to_string(res),
|
||||
std::move(data.receive.data)
|
||||
);
|
||||
}
|
||||
if (data.options.throwOnFailure && !data.receive.data.ok())
|
||||
{
|
||||
sLogRelease("errors", "get http failed " << logVar(url) << logVar(data.receive.data.statusCode));
|
||||
throw Exception (
|
||||
"HTTPConnection getFile failed code " + std::to_string(data.receive.data.statusCode),
|
||||
std::move(data.receive.data)
|
||||
);
|
||||
}
|
||||
|
||||
return data.receive.data;
|
||||
};
|
||||
|
||||
Promise<FileContents> promise;
|
||||
if (auto request = strong(request_))
|
||||
{
|
||||
sLogDebug("core::internet", "perform will consume");
|
||||
promise = std::move(request->promise);
|
||||
}
|
||||
|
||||
promise.consume(getFile, request_, connection);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace
|
||||
} // namespace
|
||||
17
tjp/core/internet/curl/Internet.hpp
Executable file
17
tjp/core/internet/curl/Internet.hpp
Executable file
@@ -0,0 +1,17 @@
|
||||
// TJP COPYRIGHT HEADER
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "../Internet.hpp"
|
||||
#include "../Internet+Internal.hpp"
|
||||
|
||||
namespace tjp::core::internet {
|
||||
|
||||
typedef void CURL;
|
||||
void perform (const WeakPtr<internal::Request> &request, CURL *connection);
|
||||
|
||||
} // namespace
|
||||
|
||||
#ifdef TJP_CORE_HEADER_ONLY
|
||||
#include "Internet.cpp"
|
||||
#endif
|
||||
126
tjp/core/internet/curl/Queue.cpp
Executable file
126
tjp/core/internet/curl/Queue.cpp
Executable file
@@ -0,0 +1,126 @@
|
||||
// TJP COPYRIGHT HEADER
|
||||
|
||||
#ifdef TJP_CORE_HEADER_ONLY
|
||||
#pragma once
|
||||
#endif
|
||||
|
||||
#include "../Queue.hpp"
|
||||
#include "Internet.hpp"
|
||||
#include <curl/curl.h>
|
||||
|
||||
#include <tjp/core/threads/Lock.hpp>
|
||||
#include <tjp/core/threads/process_queue.hpp>
|
||||
#include <thread>
|
||||
|
||||
#include <tjp/core/ptr/strong_of.hpp>
|
||||
#include <tjp/core/future/Future.hpp>
|
||||
|
||||
#include <queue>
|
||||
|
||||
//===============================================================================
|
||||
|
||||
namespace tjp::core::internet {
|
||||
|
||||
struct Queue::Internal
|
||||
{
|
||||
std::thread thread;
|
||||
bool finished = false;
|
||||
|
||||
std::queue<WeakPtr<internal::Request>> queue;
|
||||
Event event;
|
||||
Event::Mutex mutex;
|
||||
|
||||
CURL *curl;
|
||||
|
||||
void run ();
|
||||
void runOne (const WeakPtr<internal::Request> &i);
|
||||
|
||||
Internal();
|
||||
~Internal();
|
||||
} ;
|
||||
|
||||
TJP_CORE_HEADER_ONLY_INLINE
|
||||
Queue::Queue()
|
||||
{
|
||||
}
|
||||
|
||||
TJP_CORE_HEADER_ONLY_INLINE
|
||||
Queue::~Queue ()
|
||||
{
|
||||
}
|
||||
|
||||
TJP_CORE_HEADER_ONLY_INLINE
|
||||
Queue::Internal::Internal()
|
||||
{
|
||||
curl = curl_easy_init();
|
||||
}
|
||||
|
||||
TJP_CORE_HEADER_ONLY_INLINE
|
||||
Queue::Internal::~Internal()
|
||||
{
|
||||
curl_easy_cleanup(curl);
|
||||
}
|
||||
|
||||
TJP_CORE_HEADER_ONLY_INLINE
|
||||
void Queue::Internal::runOne (const WeakPtr<internal::Request> &i)
|
||||
{
|
||||
internet::perform(i, curl);
|
||||
}
|
||||
|
||||
TJP_CORE_HEADER_ONLY_INLINE
|
||||
void Queue::Internal::run ()
|
||||
{
|
||||
threads::process_queue(
|
||||
finished,
|
||||
event,
|
||||
mutex,
|
||||
queue,
|
||||
[this](auto &i) { return runOne(i); }
|
||||
);
|
||||
}
|
||||
|
||||
TJP_CORE_HEADER_ONLY_INLINE
|
||||
void Queue::prepare()
|
||||
{
|
||||
if (i)
|
||||
return;
|
||||
|
||||
static Mutex mutex;
|
||||
auto lock = lock_of(mutex);
|
||||
if (i)
|
||||
return;
|
||||
|
||||
i = strong<Internal>();
|
||||
i->thread = std::thread([s=i](){ s->run(); });
|
||||
}
|
||||
|
||||
TJP_CORE_HEADER_ONLY_INLINE
|
||||
Future<FileContents> Queue::execute(internal::RequestType type, const std::string &url, Data &&data, const HttpOptions &options)
|
||||
{
|
||||
prepare();
|
||||
|
||||
auto item = strong_of(
|
||||
internal::Request {
|
||||
.type = type,
|
||||
.url = url,
|
||||
.options = options,
|
||||
.data = std::move(data)
|
||||
}
|
||||
);
|
||||
|
||||
{
|
||||
auto lock = lock_of(i->mutex);
|
||||
i->queue.push(weak(item));
|
||||
}
|
||||
|
||||
i->event.notify_all();
|
||||
|
||||
return item->promise.get_future().then(
|
||||
[item](auto &&f) {
|
||||
return f.get();
|
||||
}
|
||||
);
|
||||
|
||||
}
|
||||
|
||||
} // namespace
|
||||
7
tjp/core/internet/curl/Queue.hpp
Executable file
7
tjp/core/internet/curl/Queue.hpp
Executable file
@@ -0,0 +1,7 @@
|
||||
// TJP COPYRIGHT HEADER
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifdef TJP_CORE_HEADER_ONLY
|
||||
#include "Queue.cpp"
|
||||
#endif
|
||||
Reference in New Issue
Block a user