Skip to content

Commit f68d349

Browse files
authored
Merge pull request #1 from jagerman/elsa
Elsa
2 parents 462f58a + 21848b0 commit f68d349

File tree

9 files changed

+84
-120
lines changed

9 files changed

+84
-120
lines changed

CMakeLists.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ endif()
1515

1616
project(spns)
1717

18-
set(CMAKE_CXX_STANDARD 17)
18+
set(CMAKE_CXX_STANDARD 20)
1919
set(CMAKE_CXX_STANDARD_REQUIRED ON)
2020
set(CMAKE_CXX_EXTENSIONS OFF)
2121
set(CMAKE_POSITION_INDEPENDENT_CODE ON)

README.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ to be installed:
8787
- python3-uwsgidecorators
8888
- python3-coloredlogs
8989
- python3-oxenmq (available in our repository)
90-
- python3-pyonionreq (available in our repository)
90+
- python3-session-util (available in our repository)
9191

9292
### Just give me some stuff to blindly copy and paste!
9393

@@ -97,7 +97,7 @@ Okay here you go (for a recent Ubuntu or Debian installation):
9797
echo "deb https://deb.oxen.io $(lsb_release -sc) main" | sudo tee /etc/apt/sources.list.d/oxen.list
9898
sudo apt update
9999
sudo apt install cmake g++ lib{sodium,oxenmq,oxenc,systemd,pq}-dev nlohmann-json3-dev \
100-
python3 python3-{systemd,flask,uwsgidecorators,coloredlogs,oxenmq,pyonionreq} \
100+
python3 python3-{systemd,flask,uwsgidecorators,coloredlogs,oxenmq,session-util} \
101101
uwsgi-plugin-python3 uwsgi-emperor
102102
```
103103

libpqxx

Submodule libpqxx updated 101 files

spns/bytes.hpp

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,9 @@ struct is_bytes_impl {
4747
template <typename T>
4848
inline constexpr bool is_bytes = decltype(is_bytes_impl::check(static_cast<T*>(nullptr)))::value;
4949

50+
template <typename T>
51+
concept bytes_subtype = is_bytes<T>;
52+
5053
struct AccountID : bytes<33> {};
5154
struct Ed25519PK : bytes<32> {};
5255
struct X25519PK : bytes<32> {};

spns/hivemind.cpp

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -912,8 +912,9 @@ void HiveMind::log_stats(std::string_view pre_cmd) {
912912
total_notifies,
913913
pushes_processed_.load());
914914

915-
auto sd_format = pre_cmd.empty() ? "STATUS={1}" : "{0}\nSTATUS={1}";
916-
sd_notify(0, fmt::format(sd_format, pre_cmd, stat_line).c_str());
915+
auto sd_out = pre_cmd.empty() ? "STATUS={}"_format(stat_line)
916+
: "{}\nSTATUS={}"_format(pre_cmd, stat_line);
917+
sd_notify(0, sd_out.c_str());
917918

918919
if (auto now = std::chrono::steady_clock::now(); now - last_stats_logged >= 4min + 55s) {
919920
log::info(stats, "Status: {}", stat_line);
@@ -1049,12 +1050,7 @@ void HiveMind::on_notifier_validation(
10491050
response["message"] = std::move(message);
10501051

10511052
sub_json_set_one_response(
1052-
std::move(replier),
1053-
final_response,
1054-
i,
1055-
remaining,
1056-
multi,
1057-
std::move(response));
1053+
std::move(replier), final_response, i, remaining, multi, std::move(response));
10581054
}
10591055

10601056
std::tuple<SwarmPubkey, std::optional<Subaccount>, int64_t, Signature, std::string, nlohmann::json>
@@ -1667,8 +1663,8 @@ SELECT
16671663
ARRAY(SELECT namespace FROM sub_namespaces WHERE subscription = id ORDER BY namespace)
16681664
FROM subscriptions
16691665
WHERE
1670-
account = {} AND service = {} AND svcid = {})"_format(
1671-
tx.quote(pubkey.id), tx.quote(service), tx.quote(service_id)));
1666+
account = $1 AND service = $2 AND svcid = $3)",
1667+
{pubkey.id, service, service_id});
16721668
int64_t id;
16731669
if (result) {
16741670
auto& [row_id, sig_ts, ns_arr] = *result;

spns/onion_request.py

Lines changed: 64 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,12 @@
44

55
from . import config
66
from .web import app
7-
87
from .subrequest import make_subrequest
98

10-
import pyonionreq
11-
12-
_junk_parser = pyonionreq.junk.Parser(
13-
privkey=config.PRIVKEYS["onionreq"].encode(), pubkey=config.PUBKEYS["onionreq"].encode()
14-
)
9+
from session_util.onionreq import OnionReqParser
1510

11+
onion_privkey_bytes = config.PRIVKEYS["onionreq"].encode()
12+
onion_pubkey_bytes = config.PUBKEYS["onionreq"].encode()
1613

1714
def bencode_consume_string(body: memoryview) -> Tuple[memoryview, memoryview]:
1815
"""
@@ -35,6 +32,64 @@ def bencode_consume_string(body: memoryview) -> Tuple[memoryview, memoryview]:
3532

3633

3734
def handle_v4_onionreq_plaintext(body):
35+
try:
36+
if not (body.startswith(b'l') and body.endswith(b'e')):
37+
raise RuntimeError("Invalid onion request body: expected bencoded list")
38+
39+
belems = memoryview(body)[1:-1]
40+
41+
# Metadata json; this element is always required:
42+
meta, belems = bencode_consume_string(belems)
43+
44+
meta = json.loads(meta.tobytes())
45+
46+
# Then we can have a second optional string containing the body:
47+
if len(belems) > 1:
48+
subreq_body, belems = bencode_consume_string(belems)
49+
if len(belems):
50+
raise RuntimeError("Invalid v4 onion request: found more than 2 parts")
51+
else:
52+
subreq_body = b''
53+
54+
method, endpoint = meta['method'], meta['endpoint']
55+
if not endpoint.startswith('/'):
56+
raise RuntimeError("Invalid v4 onion request: endpoint must start with /")
57+
58+
response, headers = make_subrequest(
59+
method, endpoint, headers=meta.get('headers', {}), body=subreq_body
60+
)
61+
62+
data = response.get_data()
63+
app.logger.debug(
64+
f"Onion sub-request for {endpoint} returned {response.status_code}, {len(data)} bytes"
65+
)
66+
67+
meta = {'code': response.status_code, 'headers': headers}
68+
69+
except Exception as e:
70+
app.logger.warning("Invalid v4 onion request: {}".format(e))
71+
meta = {'code': http.BAD_REQUEST, 'headers': {'content-type': 'text/plain; charset=utf-8'}}
72+
data = b'Invalid v4 onion request'
73+
74+
meta = json.dumps(meta).encode()
75+
return b''.join(
76+
(b'l', str(len(meta)).encode(), b':', meta, str(len(data)).encode(), b':', data, b'e')
77+
)
78+
79+
80+
def decrypt_onionreq():
81+
try:
82+
return OnionReqParser(
83+
onion_pubkey_bytes,
84+
onion_privkey_bytes,
85+
request.data)
86+
except Exception as e:
87+
app.logger.warning("Failed to decrypt onion request: {}".format(e))
88+
abort(http.BAD_REQUEST)
89+
90+
91+
@app.post("/oxen/v4/lsrpc")
92+
def handle_v4_onion_request():
3893
"""
3994
Handles a decrypted v4 onion request; this injects a subrequest to process it then returns the
4095
result of that subrequest. In contrast to v3, it is more efficient (particularly for binary
@@ -54,11 +109,6 @@ def handle_v4_onionreq_plaintext(body):
54109
needs to be accessed through a v4 request for some reason then it can be accessed via the
55110
"/legacy/whatever" endpoint).
56111
57-
If an "endpoint" contains unicode characters then it is recommended to provide it as direct
58-
UTF-8 values (rather than URL-encoded UTF-8). Both approaches will work, but the X-SOGS-*
59-
authentication headers will always apply on the final, URL-decoded value and so avoiding
60-
URL-encoding in the first place will typically simplify client implementations.
61-
62112
The "headers" field typically carries X-SOGS-* authentication headers as well as fields like
63113
Content-Type. Note that, unlike v3 requests, the Content-Type does *not* have any default and
64114
should also be specified, often as `application/json`. Unlike HTTP requests, Content-Length is
@@ -128,61 +178,6 @@ def handle_v4_onionreq_plaintext(body):
128178
bytes are returned directly to the client (i.e. no base64 encoding applied, unlike v3 requests).
129179
""" # noqa: E501
130180

131-
try:
132-
if not (body.startswith(b"l") and body.endswith(b"e")):
133-
raise RuntimeError("Invalid onion request body: expected bencoded list")
134-
135-
belems = memoryview(body)[1:-1]
136-
137-
# Metadata json; this element is always required:
138-
meta, belems = bencode_consume_string(belems)
139-
140-
meta = json.loads(meta.tobytes())
141-
142-
# Then we can have a second optional string containing the body:
143-
if len(belems) > 1:
144-
subreq_body, belems = bencode_consume_string(belems)
145-
if len(belems):
146-
raise RuntimeError("Invalid v4 onion request: found more than 2 parts")
147-
else:
148-
subreq_body = b""
149-
150-
method, endpoint = meta["method"], meta["endpoint"]
151-
if not endpoint.startswith("/"):
152-
raise RuntimeError("Invalid v4 onion request: endpoint must start with /")
153-
154-
response, headers = make_subrequest(
155-
method,
156-
endpoint,
157-
headers=meta.get("headers", {}),
158-
body=subreq_body,
159-
user_reauth=True, # Because onion requests have auth headers on the *inside*
160-
)
161-
162-
data = response.get_data()
163-
app.logger.debug(
164-
f"Onion sub-request for {endpoint} returned {response.status_code}, {len(data)} bytes"
165-
)
166-
167-
meta = {"code": response.status_code, "headers": headers}
168-
169-
except Exception as e:
170-
app.logger.warning("Invalid v4 onion request: {}".format(e))
171-
meta = {"code": 400, "headers": {"content-type": "text/plain; charset=utf-8"}}
172-
data = b"Invalid v4 onion request"
173-
174-
meta = json.dumps(meta).encode()
175-
return b"".join(
176-
(b"l", str(len(meta)).encode(), b":", meta, str(len(data)).encode(), b":", data, b"e")
177-
)
178-
179-
180-
@app.post("/oxen/v4/lsrpc")
181-
def handle_v4_onion_request():
182-
"""
183-
Parse a v4 onion request. See handle_v4_onionreq_plaintext().
184-
"""
185-
186181
# Some less-than-ideal decisions in the onion request protocol design means that we are stuck
187182
# dealing with parsing the request body here in the internal format that is meant for storage
188183
# server, but the *last* hop's decrypted, encoded data has to get shared by us (and is passed on
@@ -198,13 +193,13 @@ def handle_v4_onion_request():
198193
# The parse_junk here takes care of decoding and decrypting this according to the fields *meant
199194
# for us* in the json (which include things like the encryption type and ephemeral key):
200195
try:
201-
junk = _junk_parser.parse_junk(request.data)
196+
parser = decrypt_onionreq()
202197
except RuntimeError as e:
203198
app.logger.warning("Failed to decrypt onion request: {}".format(e))
204199
abort(400)
205200

206201
# On the way back out we re-encrypt via the junk parser (which uses the ephemeral key and
207202
# enc_type that were specified in the outer request). We then return that encrypted binary
208203
# payload as-is back to the client which bounces its way through the SN path back to the client.
209-
response = handle_v4_onionreq_plaintext(junk.payload)
210-
return junk.transformReply(response)
204+
response = handle_v4_onionreq_plaintext(parser.payload)
205+
return parser.encrypt_reply(response)

spns/pg.hpp

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
1+
#pragma once
2+
13
#include <chrono>
24
#include <mutex>
35
#include <pqxx/pqxx>
4-
#include <stack>
6+
#include <deque>
57

68
#include "bytes.hpp"
79

@@ -117,7 +119,7 @@ inline const std::string type_name<spns::Signature>{"spns::Signature"};
117119
template <>
118120
inline const std::string type_name<spns::EncKey>{"spns::EncKey"};
119121

120-
template <typename T, typename = std::enable_if_t<spns::is_bytes<T>>>
122+
template <spns::bytes_subtype T>
121123
struct spns_byte_helper {
122124
static constexpr size_t SIZE = T::SIZE;
123125
static T from_string(std::string_view text) {
@@ -148,9 +150,6 @@ struct spns_byte_helper {
148150
}
149151
};
150152

151-
template <typename T>
152-
struct nullness<T, std::enable_if_t<spns::is_bytes<T>>> : pqxx::no_null<T> {};
153-
154153
template <>
155154
struct string_traits<spns::AccountID> : spns_byte_helper<spns::AccountID> {};
156155
template <>

spns/utils.hpp

Lines changed: 2 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#pragma once
22

33
#include <fmt/format.h>
4+
#include <oxen/log/format.hpp>
45

56
#include <charconv>
67
#include <chrono>
@@ -40,37 +41,7 @@ inline ustring_view as_usv(bstring_view s) {
4041
return {reinterpret_cast<const unsigned char*>(s.data()), s.size()};
4142
}
4243

43-
// Can replace this with a `using namespace oxen::log::literals` if we start using oxen-logging
44-
namespace detail {
45-
46-
// Internal implementation of _format that holds the format temporarily until the (...) operator
47-
// is invoked on it. This object cannot be moved, copied but only used ephemerally in-place.
48-
struct fmt_wrapper {
49-
private:
50-
std::string_view format;
51-
52-
// Non-copyable and non-movable:
53-
fmt_wrapper(const fmt_wrapper&) = delete;
54-
fmt_wrapper& operator=(const fmt_wrapper&) = delete;
55-
fmt_wrapper(fmt_wrapper&&) = delete;
56-
fmt_wrapper& operator=(fmt_wrapper&&) = delete;
57-
58-
public:
59-
constexpr explicit fmt_wrapper(const char* str, const std::size_t len) : format{str, len} {}
60-
61-
/// Calling on this object forwards all the values to fmt::format, using the format string
62-
/// as provided during construction (via the "..."_format user-defined function).
63-
template <typename... T>
64-
auto operator()(T&&... args) && {
65-
return fmt::format(format, std::forward<T>(args)...);
66-
}
67-
};
68-
69-
} // namespace detail
70-
71-
inline detail::fmt_wrapper operator""_format(const char* str, size_t len) {
72-
return detail::fmt_wrapper{str, len};
73-
}
44+
using namespace oxen::log::literals;
7445

7546
/// Splits a string on some delimiter string and returns a vector of string_view's pointing into the
7647
/// pieces of the original string. The pieces are valid only as long as the original string remains

0 commit comments

Comments
 (0)