value: add Query relevant structures

This commit is contained in:
Simon Désaulniers committed 2016-07-10 19:26:04 +02:00
1 parent a533807fcf
commit 07e4435a07
6 files changed
+720 -20

No files matched your search

+3 -1
View File
@@ -21,6 +21,8 @@
#include "value.h"
MSGPACK_ADD_ENUM(dht::Value::Field);
namespace dht {
enum class ImStatus : uint8_t {
NONE = 0,
@@ -175,7 +177,7 @@ public:
pk.pack_bin_body((const char*)ice_data.data(), ice_data.size());
#else
// hack for backward compatibility with old opendht compiled with msgpack 1.0
// remove when enough people have moved to new versions
// remove when enough people have moved to new versions
pk.pack_array(ice_data.size());
for (uint8_t b : ice_data)
pk.pack(b);
+2
View File
@@ -168,4 +168,6 @@ unpackMsg(Blob b) {
msgpack::unpacked unpackMsg(Blob b);
msgpack::object* findMapValue(msgpack::object& map, const std::string& key);
} // namespace dht
+400 -5
View File
@@ -35,10 +35,12 @@
#include <functional>
#include <memory>
#include <chrono>
#include <set>
namespace dht {
struct Value;
struct Query;
/**
* A storage policy is applied once to every incoming value storage requests.
@@ -114,6 +116,14 @@ struct ValueType {
*/
struct Value
{
enum class Field {
None = 0,
Id,
ValueType,
OwnerPk,
UserType,
};
typedef uint64_t Id;
static const Id INVALID_ID {0};
@@ -135,14 +145,18 @@ struct Value
return f1(v) and f2(v);
};
}
static Filter chain(std::initializer_list<Filter> l) {
const std::vector<Filter> list(l.begin(), l.end());
return [list](const Value& v){
for (const auto& f : list)
template <typename T>
static Filter chainAll(T&& set) {
using namespace std::placeholders;
return std::bind([](const Value& v, T& s) {
for (const auto& f : s)
if (f and not f(v))
return false;
return true;
};
}, _1, std::move(set));
}
static Filter chain(std::initializer_list<Filter> l) {
return chainAll(std::move(l));
}
static Filter chainOr(Filter&& f1, Filter&& f2) {
if (not f1 or not f2) return AllFilter();
@@ -162,6 +176,11 @@ struct Value
return v.type == tid;
};
}
static Filter TypeFilter(const ValueType::Id& tid) {
return [tid](const Value& v) {
return v.type == tid;
};
}
static Filter IdFilter(const Id id) {
return [id](const Value& v) {
@@ -175,6 +194,23 @@ struct Value
};
}
static Filter ownerFilter(const crypto::PublicKey& pk) {
return ownerFilter(pk.getId());
}
static Filter ownerFilter(const InfoHash& pkh) {
return [pkh](const Value& v) {
return v.owner and v.owner->getId() == pkh;
};
}
static Filter userTypeFilter(const std::string& ut)
{
return [ut](const Value& v) {
return v.user_type == ut;
};
}
class SerializableBase
{
public:
@@ -415,6 +451,31 @@ struct Value
pk.pack(std::string("dat")); msgpack_pack_to_encrypt(pk);
}
template <typename Packer>
void msgpack_pack_fields(const std::set<Value::Field>& fields, Packer& pk) const
{
for (const auto& field : fields)
switch (field) {
case Value::Field::Id:
pk.pack(id);
break;
case Value::Field::ValueType:
pk.pack(type);
break;
case Value::Field::OwnerPk:
if (owner)
owner->msgpack_pack(pk);
else
InfoHash().msgpack_pack(pk);
break;
case Value::Field::UserType:
pk.pack(user_type);
break;
default:
break;
}
}
void msgpack_unpack(msgpack::object o);
void msgpack_unpack_body(const msgpack::object& o);
Blob getPacked() const {
@@ -424,6 +485,8 @@ struct Value
return {buffer.data(), buffer.data()+buffer.size()};
}
void msgpack_unpack_fields(const std::set<Value::Field>& fields, const msgpack::object& o, unsigned offset);
Id id {INVALID_ID};
/**
@@ -467,6 +530,338 @@ struct Value
using ValuesExport = std::pair<InfoHash, Blob>;
/**
* @class FieldValue
* @brief Describes a value filter.
* @details
* This structure holds the value for a specified field. It's type can either be
* uint64_t, InfoHash or Blob.
*/
struct FieldValue
{
FieldValue() {}
FieldValue(Value::Field f, uint64_t int_value) : field(f), intValue(int_value) {}
FieldValue(Value::Field f, InfoHash hash_value) : field(f), hashValue(hash_value) {}
FieldValue(Value::Field f, Blob blob_value) : field(f), blobValue(blob_value) {}
bool operator==(const FieldValue& fd) const;
// accessors
Value::Field getField() const { return field; }
uint64_t getInt() const { return intValue; }
InfoHash getHash() const { return hashValue; }
Blob getBlob() const { return blobValue; }
template <typename Packer>
void msgpack_pack(Packer& p) const {
p.pack_map(2);
p.pack(std::string("f")); p.pack(static_cast<uint8_t>(field));
p.pack(std::string("v"));
switch (field) {
case Value::Field::Id:
case Value::Field::ValueType:
p.pack(intValue);
break;
case Value::Field::OwnerPk:
p.pack(hashValue);
break;
case Value::Field::UserType:
p.pack_bin(blobValue.size());
p.pack_bin_body((const char*)blobValue.data(), blobValue.size());
break;
default:
throw msgpack::type_error();
}
}
void msgpack_unpack(msgpack::object msg) {
hashValue = {};
blobValue.clear();
if (auto f = findMapValue(msg, "f"))
field = (Value::Field)f->as<unsigned>();
else
throw msgpack::type_error();
auto v = findMapValue(msg, "v");
if (not v)
throw msgpack::type_error();
else
switch (field) {
case Value::Field::Id:
case Value::Field::ValueType:
intValue = v->as<decltype(intValue)>();
break;
case Value::Field::OwnerPk:
hashValue = v->as<decltype(hashValue)>();
break;
case Value::Field::UserType:
blobValue = unpackBlob(*v);
break;
default:
throw msgpack::type_error();
}
}
Value::Filter getLocalFilter() const;
private:
Value::Field field {Value::Field::None};
// three possible value types
uint64_t intValue {};
InfoHash hashValue {};
Blob blobValue {};
};
/**
* @struct FieldSelectorDescription
* @brief Describes a selection.
* @details
* This is meant to narrow data to a set of specified fields. This structure is
* used to construct a Select structure.
*/
struct FieldSelectorDescription
{
FieldSelectorDescription() {}
FieldSelectorDescription(Value::Field f) : field(f) {}
Value::Field getField() const { return field; }
bool operator==(const FieldSelectorDescription& fd) const { return field == fd.field; }
template <typename Packer>
void msgpack_pack(Packer& p) const { p.pack(static_cast<uint8_t>(field)); }
void msgpack_unpack(msgpack::object msg) { field = static_cast<Value::Field>(msg.as<int>()); }
private:
Value::Field field {Value::Field::None};
};
/**
* @class Select
* @brief Serializable Value field selection.
* @details
* This is a container for a list of FieldSelectorDescription instances. It
* describes a complete SELECT query for dht::Value.
*/
struct Select
{
Select() { }
Select(const std::string& q_str);
bool isSatisfiedBy(const Select& os) const;
/**
* Selects a field of type Value::Field.
*
* @param field the field to require.
*
* @return the resulting Select instance.
*/
Select& field(Value::Field field) {
fieldSelection_.emplace_back(field);
return *this;
}
/**
* Computes the set of selected fields based on previous require* calls.
*
* @return the set of fields.
*/
std::set<Value::Field> getSelection() const {
std::set<Value::Field> fields {};
for (const auto& f : fieldSelection_) {
fields.insert(f.getField());
}
return fields;
}
template <typename Packer>
void msgpack_pack(Packer& pk) const { pk.pack(fieldSelection_); }
void msgpack_unpack(const msgpack::object& o) {
fieldSelection_.clear();
fieldSelection_ = o.as<decltype(fieldSelection_)>();
}
friend std::ostream& operator<<(std::ostream& s, const dht::Select& q);
private:
std::vector<FieldSelectorDescription> fieldSelection_ {};
};
/**
* @class Where
* @brief Serializable dht::Value filter.
* @details
* This is container for a list of FieldValue instances. It describes a
* complete WHERE query for dht::Value.
*/
struct Where
{
Where() { }
Where(const std::string& q_str);
bool isSatisfiedBy(const Where& where) const;
/**
* Adds restriction on Value::Id based on the id argument.
*
* @param id the id.
*
* @return the resulting Where instance.
*/
Where& id(Value::Id id) {
filters_.emplace_back(Value::Field::Id, id);
return *this;
}
/**
* Adds restriction on Value::ValueType based on the type argument.
*
* @param type the value type.
*
* @return the resulting Where instance.
*/
Where& valueType(ValueType::Id type) {
filters_.emplace_back(Value::Field::ValueType, type);
return *this;
}
/**
* Adds restriction on Value::OwnerPk based on the owner_pk_hash argument.
*
* @param owner_pk_hash the owner public key fingerprint.
*
* @return the resulting Where instance.
*/
Where& owner(InfoHash owner_pk_hash) {
filters_.emplace_back(Value::Field::OwnerPk, owner_pk_hash);
return *this;
}
/**
* Adds restriction on Value::UserType based on the user_type argument.
*
* @param user_type the user type.
*
* @return the resulting Where instance.
*/
Where& userType(std::string user_type) {
filters_.emplace_back(Value::Field::UserType, Blob {user_type.begin(), user_type.end()});
return *this;
}
/**
* Computes the Value::Filter based on the list of field value set.
*
* @return the resulting Value::Filter.
*/
Value::Filter getFilter() const {
std::vector<Value::Filter> fset(filters_.size());
std::transform(filters_.begin(), filters_.end(), fset.begin(), [](const FieldValue& f) {
return f.getLocalFilter();
});
return Value::Filter::chainAll(std::move(fset));
}
template <typename Packer>
void msgpack_pack(Packer& pk) const { pk.pack(filters_); }
void msgpack_unpack(const msgpack::object& o) {
filters_.clear();
filters_ = o.as<decltype(filters_)>();
}
friend std::ostream& operator<<(std::ostream& s, const dht::Where& q);
private:
std::vector<FieldValue> filters_;
};
/**
* @class Query
* @brief Describes a query destined to another peer.
* @details
* This class describes the list of filters on field values and the field
* itselves to include in the peer response to a GET operation. See
* FieldValue.
*/
struct Query
{
static const std::string QUERY_PARSE_ERROR;
Query(Select s = {}, Where w = {}) : select(s), where(w) { };
/**
* Initializes a query based on a SQL-ish formatted string. The abstract
* form of such a string is the following:
*
* [SELECT <$field$> [WHERE <$field$=$value$>]]
*
* where
*
* - $field$ = *|id|value_type|owner_pk|user_type
* - $value$ = $string$|$integer$
* - $string$: a simple string WITHOUT SPACES.
* - $integer$: a simple integer.
*/
Query(std::string q_str) {
auto pos_W = q_str.find("WHERE");
auto pos_w = q_str.find("where");
auto pos = std::min(pos_W != std::string::npos ? pos_W : q_str.size(),
pos_w != std::string::npos ? pos_w : q_str.size());
select = q_str.substr(0, pos);
where = q_str.substr(pos, q_str.size()-pos);
}
/**
* Tell if the query is satisfied by another query.
*/
bool isSatisfiedBy(const Query& q) const;
template <typename Packer>
void msgpack_pack(Packer& pk) const {
pk.pack_map(2);
pk.pack(std::string("s")); pk.pack(select); /* packing field selectors */
pk.pack(std::string("w")); pk.pack(where); /* packing filters */
}
void msgpack_unpack(const msgpack::object& o);
friend std::ostream& operator<<(std::ostream& s, const dht::Query& q) {
s << "Query[" << q.select << " " << q.where << "]";
}
Select select {};
Where where {};
};
/*!
* @class FieldValueIndex
* @brief An index for field values.
* @details
* This structures is meant to manipulate a subset of fields normally contained
* in Value.
*/
struct FieldValueIndex {
FieldValueIndex() {}
FieldValueIndex(const Value& v, Select s = {});
/**
* Tells if all the fields of this are contained in the other
* FieldValueIndex with the same value.
*
* @param other The other FieldValueIndex instance.
*/
bool containedIn(const FieldValueIndex& other) const;
friend std::ostream& operator<<(std::ostream& os, const FieldValueIndex& fvi);
void msgpack_unpack_fields(const std::set<Value::Field>& fields,
const msgpack::object& o,
unsigned offset);
std::map<Value::Field, FieldValue> index {};
};
template <typename T,
typename std::enable_if<std::is_base_of<Value::SerializableBase, T>::value, T>::type* = nullptr>
Value::Filter
-14
View File
@@ -957,20 +957,6 @@ NetworkEngine::sendError(const sockaddr* sa,
send(buffer.data(), buffer.size(), 0, sa, salen);
}
msgpack::object*
findMapValue(msgpack::object& map, const std::string& key) {
if (map.type != msgpack::type::MAP) throw msgpack::type_error();
for (unsigned i = 0; i < map.via.map.size; i++) {
auto& o = map.via.map.ptr[i];
if(o.key.type != msgpack::type::STR)
continue;
if (o.key.as<std::string>() == key) {
return &o.val;
}
}
return nullptr;
}
void
ParsedMessage::msgpack_unpack(msgpack::object msg)
{
+11
View File
@@ -85,4 +85,15 @@ unpackMsg(Blob b) {
return msgpack::unpack((const char*)b.data(), b.size());
}
msgpack::object*
findMapValue(msgpack::object& map, const std::string& key) {
if (map.type != msgpack::type::MAP) throw msgpack::type_error();
for (unsigned i = 0; i < map.via.map.size; i++) {
auto& o = map.via.map.ptr[i];
if (o.key.type == msgpack::type::STR && o.key.as<std::string>() == key)
return &o.val;
}
return nullptr;
}
}
+304
View File
@@ -25,6 +25,8 @@
namespace dht {
const std::string Query::QUERY_PARSE_ERROR {"Error parsing query."};
std::ostream& operator<< (std::ostream& s, const Value& v)
{
s << "Value[id:" << std::hex << v.id << std::dec << " ";
@@ -157,4 +159,306 @@ Value::msgpack_unpack_body(const msgpack::object& o)
}
}
bool
FieldValue::operator==(const FieldValue& vfd) const
{
if (field != vfd.field)
return false;
switch (field) {
case Value::Field::Id:
case Value::Field::ValueType:
return intValue == vfd.intValue;
case Value::Field::OwnerPk:
return hashValue == vfd.hashValue;
case Value::Field::UserType:
return blobValue == vfd.blobValue;
case Value::Field::None:
return true;
default:
return false;
}
}
Value::Filter
FieldValue::getLocalFilter() const
{
switch (field) {
case Value::Field::Id:
return Value::IdFilter(intValue);
case Value::Field::ValueType:
return Value::TypeFilter(intValue);
case Value::Field::OwnerPk:
return Value::ownerFilter(hashValue);
case Value::Field::UserType:
return Value::userTypeFilter(std::string {blobValue.begin(), blobValue.end()});
default:
return Value::AllFilter();
}
}
FieldValueIndex::FieldValueIndex(const Value& v, Select s)
{
auto selection = s.getSelection();
if (not selection.empty()) {
std::transform(selection.begin(), selection.end(), std::inserter(index, index.end()),
[](const std::set<Value::Field>::value_type& f) {
return std::make_pair(f, FieldValue {});
});
} else {
index.clear();
for (size_t f = 1 ; f < 5 ; ++f)
index[static_cast<Value::Field>(f)] = {};
}
for (const auto& fvp : index) {
const auto& f = fvp.first;
switch (f) {
case Value::Field::Id:
index[f] = {f, v.id};
break;
case Value::Field::ValueType:
index[f] = {f, v.type};
break;
case Value::Field::OwnerPk:
index[f] = {f, v.owner ? v.owner->getId() : InfoHash() };
break;
case Value::Field::UserType:
index[f] = {f, Blob {v.user_type.begin(), v.user_type.end()}};
break;
default:
break;
}
}
}
bool FieldValueIndex::containedIn(const FieldValueIndex& other) const {
if (index.size() > other.index.size())
return false;
for (const auto& field : index) {
auto other_field = other.index.find(field.first);
if (other_field == other.index.end())
return false;
}
return true;
}
std::ostream& operator<<(std::ostream& os, const FieldValueIndex& fvi) {
os << "Index[";
for (auto v = fvi.index.begin(); v != fvi.index.end(); ++v) {
switch (v->first) {
case Value::Field::Id:
os << "Id:" << std::hex << v->second.getInt();
break;
case Value::Field::ValueType:
os << "ValueType:" << v->second.getInt();
break;
case Value::Field::OwnerPk:
os << "Owner:" << v->second.getHash().toString();
break;
case Value::Field::UserType: {
auto ut = v->second.getBlob();
os << "UserType:" << std::string(ut.begin(), ut.end());
break;
}
default:
break;
}
os << (std::next(v) != fvi.index.end() ? "," : "");
}
return os << "]";
}
void
FieldValueIndex::msgpack_unpack_fields(const std::set<Value::Field>& fields, const msgpack::object& o, unsigned offset)
{
index.clear();
unsigned j = 0;
for (const auto& field : fields) {
auto& field_value = o.via.array.ptr[offset+(j++)];
switch (field) {
case Value::Field::Id:
case Value::Field::ValueType:
index[field] = FieldValue(field, field_value.as<uint64_t>());
break;
case Value::Field::OwnerPk:
index[field] = FieldValue(field, field_value.as<InfoHash>());
break;
case Value::Field::UserType:
index[field] = FieldValue(field, field_value.as<Blob>());
break;
default:
throw msgpack::type_error();
}
}
}
void trim_str(std::string& str) {
auto first = std::min(str.size(), str.find_first_not_of(" "));
auto last = std::min(str.size(), str.find_last_not_of(" "));
str = str.substr(first, last - first + 1);
}
Select::Select(const std::string& q_str) {
std::istringstream q_iss {q_str};
std::string token {};
q_iss >> token;
if (token == "SELECT" or token == "select") {
q_iss >> token;
std::istringstream fields {token};
while (std::getline(fields, token, ',')) {
trim_str(token);
if (token == "id")
field(Value::Field::Id);
else if (token == "value_type")
field(Value::Field::ValueType);
else if (token == "owner_pk")
field(Value::Field::OwnerPk);
else if (token == "user_type")
field(Value::Field::UserType);
}
}
}
Where::Where(const std::string& q_str) {
std::istringstream q_iss {q_str};
std::string token {};
q_iss >> token;
if (token == "WHERE" or token == "where") {
std::getline(q_iss, token);
std::istringstream restrictions {token};
while (std::getline(restrictions, token, ',')) {
trim_str(token);
std::istringstream eq_ss {token};
std::string field_str, value_str;
std::getline(eq_ss, field_str, '=');
trim_str(field_str);
std::getline(eq_ss, value_str, '=');
trim_str(value_str);
if (not value_str.empty()) {
uint64_t v = 0;
std::string s {};
std::istringstream convert {value_str};
convert >> v;
if (convert.failbit and value_str.size() > 1 and value_str[0] == '\"' and value_str[value_str.size()-1] == '\"')
s = value_str.substr(1, value_str.size()-2);
else
s = value_str;
if (field_str == "id")
id(v);
else if (field_str == "value_type")
valueType(v);
else if (field_str == "owner_pk")
owner(InfoHash(s));
else if (field_str == "user_type")
userType(s);
else
throw std::invalid_argument(Query::QUERY_PARSE_ERROR + " (WHERE) wrong token near: " + field_str);
}
}
}
}
void
Query::msgpack_unpack(const msgpack::object& o)
{
if (o.type != msgpack::type::MAP)
throw msgpack::type_error();
auto rfilters = findMapValue(o, "w"); /* unpacking filters */
if (rfilters)
where.msgpack_unpack(*rfilters);
else
throw msgpack::type_error();
auto rfield_selector = findMapValue(o, "s"); /* unpacking field selectors */
if (rfield_selector)
select.msgpack_unpack(*rfield_selector);
else
throw msgpack::type_error();
}
template <typename T>
bool subset(std::vector<T> fds, std::vector<T> qfds)
{
for (auto& fd : fds) {
auto correspondance = std::find_if(qfds.begin(), qfds.end(), [&fd](T& _vfd) { return fd == _vfd; });
if (correspondance == qfds.end())
return false;
}
return true;
};
bool Select::isSatisfiedBy(const Select& os) const {
/* empty, means all values are selected. */
if (fieldSelection_.empty() and not os.fieldSelection_.empty())
return false;
else
return subset(fieldSelection_, os.fieldSelection_);
}
bool Where::isSatisfiedBy(const Where& ow) const {
return subset(ow.filters_, filters_);
}
bool Query::isSatisfiedBy(const Query& q) const {
return where.isSatisfiedBy(q.where) and select.isSatisfiedBy(q.select);
}
std::ostream& operator<<(std::ostream& s, const dht::Select& select) {
s << "SELECT " << (select.fieldSelection_.empty() ? "*" : "");
for (auto fs = select.fieldSelection_.begin() ; fs != select.fieldSelection_.end() ; ++fs) {
switch (fs->getField()) {
case Value::Field::Id:
s << "id";
break;
case Value::Field::ValueType:
s << "value_type";
break;
case Value::Field::UserType:
s << "user_type";
break;
case Value::Field::OwnerPk:
s << "owner_public_key";
break;
default:
break;
}
s << (std::next(fs) != select.fieldSelection_.end() ? "," : "");
}
return s;
}
std::ostream& operator<<(std::ostream& s, const dht::Where& where) {
if (not where.filters_.empty()) {
s << "WHERE ";
for (auto f = where.filters_.begin() ; f != where.filters_.end() ; ++f) {
switch (f->getField()) {
case Value::Field::Id:
s << "id=" << f->getInt();
break;
case Value::Field::ValueType:
s << "value_type=" << f->getInt();
break;
case Value::Field::OwnerPk:
s << "owner_pk_hash=" << f->getHash().toString();
break;
case Value::Field::UserType: {
auto b = f->getBlob();
s << "user_type=" << std::string {b.begin(), b.end()};
break;
}
default:
break;
}
s << (std::next(f) != where.filters_.end() ? "," : "");
}
}
return s;
}
}