From 0faf9cbb56f05498923de8206e3556d233ba10a0 Mon Sep 17 00:00:00 2001 From: Amnon Heiman Date: Wed, 11 Mar 2015 14:11:06 +0200 Subject: [PATCH 01/12] Adding back, find_last_of and append to sstring This adds the back method that return a reference or const reference to the last char in an sstring. find_last_of, which return the index of the last occurance of a char in the string. And append with append C string to a string. The logic and definition are similiar to the std::string. Note that following the std::string definition, calling back on an empty string is forbiden and the results are undefined. Signed-off-by: Amnon Heiman --- core/sstring.hh | 57 +++++++++++++++++++++++++++++++++++++++++++ tests/sstring_test.cc | 16 ++++++++++++ 2 files changed, 73 insertions(+) diff --git a/core/sstring.hh b/core/sstring.hh index 5d688e2d52..db89311df9 100644 --- a/core/sstring.hh +++ b/core/sstring.hh @@ -209,6 +209,63 @@ public: return npos; } + /** + * find_last_of find the last occurrence of c in the string. + * When pos is specified, the search only includes characters + * at or before position pos. + * + */ + size_t find_last_of (char_type c, size_t pos = npos) const noexcept { + const char_type* str_start = str(); + if (size()) { + if (pos >= size()) { + pos = size() - 1; + } + const char_type* p = str_start + pos + 1; + do { + p--; + if (*p == c) { + return (p - str_start); + } + } while (p != str_start); + } + return npos; + } + + /** + * Append a C substring. + * @param s The C string to append. + * @param n The number of characters to append. + * @return Reference to this string. + */ + basic_sstring& append (const char_type* s, size_t n) { + basic_sstring ret(initialized_later(), size() + n); + std::copy(begin(), end(), ret.begin()); + std::copy(s, s + n, ret.begin() + size()); + *this = ret; + return *this; + } + + /** + * Returns a read/write reference to the data at the last + * element of the string. + * This function shall not be called on empty strings. + */ + reference + back() noexcept { + return operator[](size() - 1); + } + + /** + * Returns a read-only (constant) reference to the data at the last + * element of the string. + * This function shall not be called on empty strings. + */ + const_reference + back() const noexcept { + return operator[](size() - 1); + } + basic_sstring substr(size_t from, size_t len = npos) const { if (from > size()) { throw std::out_of_range("sstring::substr out of range"); diff --git a/tests/sstring_test.cc b/tests/sstring_test.cc index ab3645d868..c280fdcabc 100644 --- a/tests/sstring_test.cc +++ b/tests/sstring_test.cc @@ -75,3 +75,19 @@ BOOST_AUTO_TEST_CASE(test_at_sstring) { s.at(1) = 'd'; BOOST_REQUIRE_EQUAL(s, "adcde"); } + +BOOST_AUTO_TEST_CASE(test_find_last_sstring) { + BOOST_REQUIRE_EQUAL(sstring("ababa").find_last_of('a'), 4); + BOOST_REQUIRE_EQUAL(sstring("ababa").find_last_of('a',5), 4); + BOOST_REQUIRE_EQUAL(sstring("ababa").find_last_of('a',4), 4); + BOOST_REQUIRE_EQUAL(sstring("ababa").find_last_of('a',3), 2); + BOOST_REQUIRE_EQUAL(sstring("ababa").find_last_of('x'), sstring::npos); + BOOST_REQUIRE_EQUAL(sstring("").find_last_of('a'), sstring::npos); +} + + +BOOST_AUTO_TEST_CASE(test_append) { + BOOST_REQUIRE_EQUAL(sstring("aba").append("1234", 3), "aba123"); + BOOST_REQUIRE_EQUAL(sstring("aba").append("1234", 4), "aba1234"); + BOOST_REQUIRE_EQUAL(sstring("aba").append("1234", 0), "aba"); +} From c420b75e26934c0d4af02e34ea306fb241872231 Mon Sep 17 00:00:00 2001 From: Amnon Heiman Date: Sun, 15 Mar 2015 16:21:58 +0200 Subject: [PATCH 02/12] remove leading spaces from headers --- http/request_parser.rl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http/request_parser.rl b/http/request_parser.rl index 61d7008e91..b06c82b65b 100644 --- a/http/request_parser.rl +++ b/http/request_parser.rl @@ -89,7 +89,7 @@ http_version = 'HTTP/' (digit '.' digit) >mark %store_version; field = tchar+ >mark %store_field_name; value = any* >mark %store_value; start_line = ((operation sp uri sp http_version) -- crlf) crlf; -header_1st = (field sp_ht* ':' value :> crlf) %assign_field; +header_1st = (field sp_ht* ':' sp_ht* value :> crlf) %assign_field; header_cont = (sp_ht+ value sp_ht* crlf) %extend_field; header = header_1st header_cont*; main := start_line header* :> (crlf @done); From 950d921df30ce86c02d22a4f60482480a268f696 Mon Sep 17 00:00:00 2001 From: Amnon Heiman Date: Sun, 22 Mar 2015 14:24:52 +0200 Subject: [PATCH 03/12] Http server handlers to use future The http server handlers can sometimes need to perform async operation, specifically, when reading files from the disk. To support async behavior the handlers handle function will return a future, that will be propagate back, when the future will be ready it will return the reply to be sent. When switching from direct function call to future, there is also a need to switch from references to pointers. Note that while the request will be discarded, the reply will be propagate back via the future to the caller. Signed-off-by: Amnon Heiman --- apps/httpd/main.cc | 17 +++++++++----- http/handlers.hh | 5 ++-- http/httpd.hh | 24 +++++++++++-------- http/routes.cc | 32 ++++++++++++++++---------- http/routes.hh | 3 ++- tests/httpd.cc | 57 +++++++++++++++++++++++++++++++--------------- 6 files changed, 90 insertions(+), 48 deletions(-) diff --git a/apps/httpd/main.cc b/apps/httpd/main.cc index 8bc481191e..5d4c034983 100644 --- a/apps/httpd/main.cc +++ b/apps/httpd/main.cc @@ -28,13 +28,19 @@ using namespace httpd; class handl : public httpd::handler_base { public: - virtual void handle(const sstring& path, parameters* params, - httpd::const_req& req, httpd::reply& rep) { - rep._content = "hello"; - rep.done("html"); + virtual future > handle(const sstring& path, + std::unique_ptr req, std::unique_ptr rep) { + rep->_content = "hello"; + rep->done("html"); + return make_ready_future>(std::move(rep)); } }; +void set_routes(routes& r) { + handl* h1 = new handl(); + r.add(operation_type::GET, url("/"), h1); +} + int main(int ac, char** av) { app_template app; app.add_options()("port", bpo::value()->default_value(10000), @@ -46,8 +52,7 @@ int main(int ac, char** av) { auto server = new distributed; server->start().then([server = std::move(server), port] () mutable { server->invoke_on_all([](http_server& server) { - handl* h1 = new handl(); - server._routes.add(operation_type::GET, url("/"), h1); + set_routes(server._routes); }); server->invoke_on_all(&http_server::listen, ipv4_addr {port}); }).then([port] { diff --git a/http/handlers.hh b/http/handlers.hh index b20633cfd7..9a9888af61 100644 --- a/http/handlers.hh +++ b/http/handlers.hh @@ -25,6 +25,7 @@ #include "request.hh" #include "common.hh" #include "reply.hh" +#include "core/future-util.hh" #include @@ -48,8 +49,8 @@ public: * @param req the original request * @param rep the reply */ - virtual void handle(const sstring& path, parameters* params, - httpd::const_req& req, httpd::reply& rep) = 0; + virtual future > handle(const sstring& path, + std::unique_ptr req, std::unique_ptr rep) = 0; virtual ~handler_base() = default; diff --git a/http/httpd.hh b/http/httpd.hh index ab1ba7cfed..2b58837541 100644 --- a/http/httpd.hh +++ b/http/httpd.hh @@ -140,16 +140,19 @@ public: } future<> read_one() { _parser.init(); - return _read_buf.consume(_parser).then([this] { + return _read_buf.consume(_parser).then([this] () mutable { if (_parser.eof()) { _done = true; return make_ready_future<>(); } ++_server._requests_served; - _req = _parser.get_parsed_request(); - return _replies.not_full().then([this] { - _done = generate_reply(std::move(_req)); - }); + std::unique_ptr req = _parser.get_parsed_request(); + + return _replies.not_full().then([req = std::move(req), this] () mutable { + return generate_reply(std::move(req)); + }).then([this](bool done) { + _done = done; + }); }); } future<> respond() { @@ -199,7 +202,7 @@ public: return write_reply_headers(++hi); }); } - bool generate_reply(std::unique_ptr req) { + future generate_reply(std::unique_ptr req) { auto resp = std::make_unique(); bool conn_keep_alive = false; bool conn_close = false; @@ -226,10 +229,13 @@ public: // HTTP/0.9 goes here should_close = true; } - _server._routes.handle(req->_url, *(req.get()), *(resp.get())); + sstring url = req->_url; + return _server._routes.handle(url, std::move(req), std::move(resp)). // Caller guarantees enough room - _replies.push(std::move(resp)); - return should_close; + then([this, should_close](std::unique_ptr rep) { + this->_replies.push(std::move(rep)); + return make_ready_future(should_close); + }); } future<> write_body() { return _write_buf.write(_resp->_content.begin(), diff --git a/http/routes.cc b/http/routes.cc index 95a3f7e97a..1d7d3e179c 100644 --- a/http/routes.cc +++ b/http/routes.cc @@ -47,35 +47,41 @@ routes::~routes() { } -void routes::handle(const sstring& path, request& req, reply& rep) { - handler_base* handler = get_handler(str2type(req._method), - normalize_url(path), req.param); +future > routes::handle(const sstring& path, std::unique_ptr req, std::unique_ptr rep) { + handler_base* handler = get_handler(str2type(req->_method), + normalize_url(path), req->param); if (handler != nullptr) { try { for (auto& i : handler->_mandatory_param) { - verify_param(req, i); + verify_param(*req.get(), i); } - handler->handle(path, &req.param, req, rep); + auto r = handler->handle(path, std::move(req), std::move(rep)); + return r; } catch (const redirect_exception& _e) { - rep.add_header("Location", _e.url).set_status(_e.status()).done( + rep.reset(new reply()); + rep->add_header("Location", _e.url).set_status(_e.status()).done( "json"); - return; + } catch (const base_exception& _e) { + rep.reset(new reply()); json_exception e(_e); - rep.set_status(_e.status(), e.to_json()).done("json"); + rep->set_status(_e.status(), e.to_json()).done("json"); } catch (exception& _e) { + rep.reset(new reply()); json_exception e(_e); cerr << "exception was caught for " << path << ": " << _e.what() << endl; - rep.set_status(reply::status_type::internal_server_error, + rep->set_status(reply::status_type::internal_server_error, e.to_json()).done("json"); - return; } } else { + rep.reset(new reply()); json_exception ex(not_found_exception("Not found")); - rep.set_status(reply::status_type::not_found, ex.to_json()).done( + rep->set_status(reply::status_type::not_found, ex.to_json()).done( "json"); } + cerr << "Failed with " << path << " " << rep->_content << endl; + return make_ready_future>(std::move(rep)); } sstring routes::normalize_url(const sstring& url) { @@ -107,7 +113,9 @@ routes& routes::add(operation_type type, const url& url, handler_base* handler) { match_rule* rule = new match_rule(handler); rule->add_str(url._path); - rule->add_param(url._param, true); + if (url._param != "") { + rule->add_param(url._param, true); + } return add(rule, type); } diff --git a/http/routes.hh b/http/routes.hh index 8d02fd65fe..b3d4f268d0 100644 --- a/http/routes.hh +++ b/http/routes.hh @@ -30,6 +30,7 @@ #include #include #include +#include "core/future-util.hh" namespace httpd { @@ -125,7 +126,7 @@ public: * @param req the http request * @param rep the http reply */ - void handle(const sstring& path, httpd::request& req, httpd::reply& rep); + future > handle(const sstring& path, std::unique_ptr req, std::unique_ptr rep); private: diff --git a/tests/httpd.cc b/tests/httpd.cc index 10d49f608c..cda213a991 100644 --- a/tests/httpd.cc +++ b/tests/httpd.cc @@ -18,10 +18,10 @@ using namespace httpd; class handl : public httpd::handler_base { public: - virtual void handle(const sstring& path, parameters* params, - httpd::const_req& req, httpd::reply& rep) - { - + virtual future > handle(const sstring& path, + std::unique_ptr req, std::unique_ptr rep) { + rep->done("html"); + return make_ready_future>(std::move(rep)); } }; @@ -77,24 +77,45 @@ BOOST_AUTO_TEST_CASE(test_formatter) } - -BOOST_AUTO_TEST_CASE(test_routes) -{ +BOOST_AUTO_TEST_CASE(test_routes) { handl* h1 = new handl(); handl* h2 = new handl(); routes route; route.add(operation_type::GET, url("/api").remainder("path"), h1); route.add(operation_type::GET, url("/"), h2); - request req; - reply rep; - BOOST_CHECK_NO_THROW(route.handle("/api", req, rep)); - BOOST_REQUIRE_EQUAL((int)rep._status, (int)reply::status_type::ok); - BOOST_REQUIRE_EQUAL(req.param["path"], ""); - BOOST_CHECK_NO_THROW(route.handle("/", req, rep)); - BOOST_REQUIRE_EQUAL((int)rep._status, (int)reply::status_type::ok); - BOOST_CHECK_NO_THROW(route.handle("/api/abc", req, rep)); - BOOST_REQUIRE_EQUAL(req.param["path"], "/abc"); - BOOST_CHECK_NO_THROW(route.handle("/ap", req, rep)); - BOOST_REQUIRE_EQUAL((int)rep._status, (int)reply::status_type::not_found); + std::unique_ptr req = std::make_unique(); + std::unique_ptr rep = std::make_unique(); + + BOOST_CHECK_NO_THROW( + route.handle("/api", std::move(req), std::move(rep)).then( + [&rep](std::unique_ptr _rep) { + rep = std::move(_rep); + })); + BOOST_REQUIRE_EQUAL((int )rep->_status, (int )reply::status_type::ok); + req.reset(new request); + rep.reset(new reply); + + BOOST_CHECK_NO_THROW( + route.handle("/", std::move(req), std::move(rep)).then( + [&rep](std::unique_ptr _rep) { + rep = std::move(_rep); + })); + BOOST_REQUIRE_EQUAL((int )rep->_status, (int )reply::status_type::ok); + req.reset(new request); + rep.reset(new reply); + BOOST_CHECK_NO_THROW( + route.handle("/api/abc", std::move(req), std::move(rep)).then( + [&rep](std::unique_ptr _rep) { + rep = std::move(_rep); + })); + req.reset(new request); + rep.reset(new reply); + BOOST_CHECK_NO_THROW( + route.handle("/ap", std::move(req), std::move(rep)).then( + [&rep](std::unique_ptr _rep) { + rep = std::move(_rep); + })); + BOOST_REQUIRE_EQUAL((int )rep->_status, + (int )reply::status_type::not_found); } From be33b31ae23156661480d996d22ad1a31836e088 Mon Sep 17 00:00:00 2001 From: Amnon Heiman Date: Sun, 22 Mar 2015 14:31:37 +0200 Subject: [PATCH 04/12] Httpd Adding function handlers Most of the time, implementing an http handler consist of a small amount of logic. Function handlers are a simplified way of adding such a logic, they accept a lambda expression of various types and eliminate the need to create a type for the handlers. The prefered way of creating a handler is by using the json_request_function, it would use auto-boxing to return a json object. Signed-off-by: Amnon Heiman --- apps/httpd/main.cc | 5 ++- http/function_handlers.hh | 86 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 http/function_handlers.hh diff --git a/apps/httpd/main.cc b/apps/httpd/main.cc index 5d4c034983..3100225dee 100644 --- a/apps/httpd/main.cc +++ b/apps/httpd/main.cc @@ -21,6 +21,7 @@ #include "http/httpd.hh" #include "http/handlers.hh" +#include "http/function_handlers.hh" namespace bpo = boost::program_options; @@ -37,7 +38,9 @@ public: }; void set_routes(routes& r) { - handl* h1 = new handl(); + function_handler* h1 = new function_handler([](const_req req) { + return "hello"; + }); r.add(operation_type::GET, url("/"), h1); } diff --git a/http/function_handlers.hh b/http/function_handlers.hh new file mode 100644 index 0000000000..4d7774ad73 --- /dev/null +++ b/http/function_handlers.hh @@ -0,0 +1,86 @@ +/* + * This file is open source software, licensed to you under the terms + * of the Apache License, Version 2.0 (the "License"). See the NOTICE file + * distributed with this work for additional information regarding copyright + * ownership. You may not use this file except in compliance with the License. + * + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Copyright 2015 Cloudius Systems + */ + +#pragma once + +#include "handlers.hh" +#include +#include "json/json_elements.hh" + +namespace httpd { + +/** + * A request function is a lambda expression that gets only the request + * as its parameter + */ +typedef std::function request_function; + +/** + * A handle function is a lambda expression that gets request and reply + */ +typedef std::function handle_function; + +/** + * A json request function is a lambda expression that gets only the request + * as its parameter and return a json response. + * Using the json response is done implicitly. + */ +typedef std::function json_request_function; + +/** + * The function handler get a lambda expression in the constructor. + * it will call that expression to get the result + * This is suited for very simple handlers + * + */ +class function_handler : public handler_base { +public: + + function_handler(const handle_function & f_handle, const sstring& type) + : _f_handle(f_handle), _type(type) { + } + + function_handler(const request_function & _handle, const sstring& type) + : _f_handle([_handle](const_req req, reply& rep) { + return _handle(req); + }), _type(type) { + } + + function_handler(const json_request_function& _handle) + : _f_handle([_handle](const_req req, reply& rep) { + json::json_return_type res = _handle(req); + return res._res; + }), _type("json") { + + } + future> handle(const sstring& path, + std::unique_ptr req, std::unique_ptr rep) override { + rep->_content += _f_handle(*req.get(), *rep.get()); + rep->done(_type); + return make_ready_future>(std::move(rep)); + } + +protected: + handle_function _f_handle; + sstring _type; +}; + +} From 6512d7f7fb08a4b8b083be1f0600556bb7c40064 Mon Sep 17 00:00:00 2001 From: Amnon Heiman Date: Thu, 19 Mar 2015 12:02:07 +0200 Subject: [PATCH 05/12] http support query parameter and url encoding Http url encode query parameters by adding a question mark after the uri with pairs of key=value items. All values in the url are url decoded. This add the url encoding and query parameters support Signed-off-by: Amnon Heiman --- http/httpd.hh | 92 ++++++++++++++++++++++++++++++++++++++++++++++++-- tests/httpd.cc | 9 +++++ 2 files changed, 98 insertions(+), 3 deletions(-) diff --git a/http/httpd.hh b/http/httpd.hh index 2b58837541..4d5ccb6c27 100644 --- a/http/httpd.hh +++ b/http/httpd.hh @@ -26,6 +26,7 @@ #include "http/request.hh" #include "core/reactor.hh" #include "core/sstring.hh" +#include #include "core/app-template.hh" #include "core/circular_buffer.hh" #include "core/distributed.hh" @@ -86,7 +87,7 @@ public: try { f.get(); } catch (std::exception& ex) { - std::cout << "request error " << ex.what() << "\n"; + std::cerr << "request error " << ex.what() << std::endl; } }); do_accepts(which); @@ -94,7 +95,7 @@ public: try { f.get(); } catch (std::exception& ex) { - std::cout << "accept failed: " << ex.what() << "\n"; + std::cerr << "accept failed: " << ex.what() << std::endl; } }); } @@ -202,6 +203,91 @@ public: return write_reply_headers(++hi); }); } + + static short hex_to_byte(char c) { + if (c >='a' && c <= 'z') { + return c - 'a' + 10; + } else if (c >='A' && c <= 'Z') { + return c - 'A' + 10; + } + return c - '0'; + } + + /** + * Convert a hex encoded 2 bytes substring to char + */ + static char hexstr_to_char(const std::experimental::string_view& in, size_t from) { + + return static_cast(hex_to_byte(in[from]) * 16 + hex_to_byte(in[from + 1])); + } + + /** + * URL_decode a substring and place it in the given out sstring + */ + static bool url_decode(const std::experimental::string_view& in, sstring& out) { + size_t pos = 0; + char buff[in.length()]; + for (size_t i = 0; i < in.length(); ++i) { + if (in[i] == '%') { + if (i + 3 <= in.size()) { + buff[pos++] = hexstr_to_char(in, i + 1); + i += 2; + } else { + return false; + } + } else if (in[i] == '+') { + buff[pos++] = ' '; + } else { + buff[pos++] = in[i]; + } + } + out = sstring(buff, pos); + return true; + } + + /** + * Add a single query parameter to the parameter list + */ + static void add_param(request& req, const std::experimental::string_view& param) { + size_t split = param.find('='); + + if (split >= param.length() - 1) { + sstring key; + if (url_decode(param.substr(0,split) , key)) { + req.query_parameters[key] = ""; + } + } else { + sstring key; + sstring value; + if (url_decode(param.substr(0,split), key) + && url_decode(param.substr(split + 1), value)) { + req.query_parameters[key] = value; + } + } + + } + + /** + * Set the query parameters in the request objects. + * query param appear after the question mark and are separated + * by the ampersand sign + */ + static sstring set_query_param(request& req) { + size_t pos = req._url.find('?'); + if (pos == sstring::npos) { + return req._url; + } + size_t curr = pos + 1; + size_t end_param; + std::experimental::string_view url = req._url; + while ((end_param = req._url.find('&', curr)) != sstring::npos) { + add_param(req, url.substr(curr, end_param - curr) ); + curr = end_param + 1; + } + add_param(req, url.substr(pos + 1)); + return req._url.substr(0, pos); + } + future generate_reply(std::unique_ptr req) { auto resp = std::make_unique(); bool conn_keep_alive = false; @@ -229,7 +315,7 @@ public: // HTTP/0.9 goes here should_close = true; } - sstring url = req->_url; + sstring url = set_query_param(*req.get()); return _server._routes.handle(url, std::move(req), std::move(resp)). // Caller guarantees enough room then([this, should_close](std::unique_ptr rep) { diff --git a/tests/httpd.cc b/tests/httpd.cc index cda213a991..e38a869b7c 100644 --- a/tests/httpd.cc +++ b/tests/httpd.cc @@ -7,6 +7,7 @@ #include +#include "http/httpd.hh" #include "http/handlers.hh" #include "http/matcher.hh" #include "http/matchrules.hh" @@ -77,6 +78,14 @@ BOOST_AUTO_TEST_CASE(test_formatter) } +BOOST_AUTO_TEST_CASE(test_decode_url) { + request req; + req._url = "/a?q=%23%24%23"; + sstring url = http_server::connection::set_query_param(req); + BOOST_REQUIRE_EQUAL(url, "/a"); + BOOST_REQUIRE_EQUAL(req.get_query_param("q"), "#$#"); +} + BOOST_AUTO_TEST_CASE(test_routes) { handl* h1 = new handl(); handl* h2 = new handl(); From cb535f1fa9068ec8fce5dba7b326a420b320d2ef Mon Sep 17 00:00:00 2001 From: Amnon Heiman Date: Sun, 22 Mar 2015 19:38:00 +0200 Subject: [PATCH 06/12] Add file_type method to the reactor This method check the file type and retrun an optional value, if the file does not exists no value is return. On other errors an exception will be thrown. Signed-off-by: Amnon Heiman --- core/reactor.cc | 39 +++++++++++++++++++++++++++++++++++++++ core/reactor.hh | 1 + 2 files changed, 40 insertions(+) diff --git a/core/reactor.cc b/core/reactor.cc index 1533223c68..b0036cd802 100644 --- a/core/reactor.cc +++ b/core/reactor.cc @@ -392,6 +392,45 @@ reactor::open_file_dma(sstring name, open_flags flags) { }); } +directory_entry_type stat_to_entry_type(__mode_t type) { + if (S_ISDIR(type)) { + return directory_entry_type::directory; + } + if (S_ISBLK(type)) { + return directory_entry_type::block_device; + } + if (S_ISCHR(type)) { + return directory_entry_type::char_device; + } + if (S_ISFIFO(type)) { + return directory_entry_type::fifo; + } + if (S_ISLNK(type)) { + return directory_entry_type::link; + } + return directory_entry_type::regular; + +} + +future> +reactor::file_type(sstring name) { + return _thread_pool.submit>([name] { + struct stat st; + auto ret = stat(name.c_str(), &st); + return wrap_syscall(ret, st); + }).then([] (syscall_result_extra sr) { + if (long(sr.result) == -1) { + if (sr.result != ENOENT && sr.result != ENOTDIR) { + sr.throw_if_error(); + } + return make_ready_future > + (std::experimental::optional() ); + } + return make_ready_future > + (std::experimental::optional(stat_to_entry_type(sr.extra.st_mode)) ); + }); +} + future reactor::open_directory(sstring name) { return _thread_pool.submit>([name] { diff --git a/core/reactor.hh b/core/reactor.hh index fbb5143871..f372328841 100644 --- a/core/reactor.hh +++ b/core/reactor.hh @@ -749,6 +749,7 @@ public: future open_file_dma(sstring name, open_flags flags); future open_directory(sstring name); + future> file_type(sstring name); template future submit_io(Func prepare_io); From 2ff0f41cfba9f5319c0af796c00a0d23f0a2cc8b Mon Sep 17 00:00:00 2001 From: Amnon Heiman Date: Thu, 19 Mar 2015 12:05:49 +0200 Subject: [PATCH 07/12] http adding file handler File handlers support reading local file and return it as the result for the http request. There are two kind of handler, a file handler will return a specific file when called. A directory handler expect to find a path parameter called 'path' and would concatinate this parameter to the direcoty path. Signed-off-by: Amnon Heiman --- apps/httpd/main.cc | 3 + http/file_handler.cc | 129 ++++++++++++++++++++++++++++++++++ http/file_handler.hh | 161 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 293 insertions(+) create mode 100644 http/file_handler.cc create mode 100644 http/file_handler.hh diff --git a/apps/httpd/main.cc b/apps/httpd/main.cc index 3100225dee..0523a1b897 100644 --- a/apps/httpd/main.cc +++ b/apps/httpd/main.cc @@ -22,6 +22,7 @@ #include "http/httpd.hh" #include "http/handlers.hh" #include "http/function_handlers.hh" +#include "http/file_handler.hh" namespace bpo = boost::program_options; @@ -42,6 +43,8 @@ void set_routes(routes& r) { return "hello"; }); r.add(operation_type::GET, url("/"), h1); + r.add(operation_type::GET, url("/file").remainder("path"), + new directory_handler("/")); } int main(int ac, char** av) { diff --git a/http/file_handler.cc b/http/file_handler.cc new file mode 100644 index 0000000000..e5a3b82253 --- /dev/null +++ b/http/file_handler.cc @@ -0,0 +1,129 @@ +/* + * This file is open source software, licensed to you under the terms + * of the Apache License, Version 2.0 (the "License"). See the NOTICE file + * distributed with this work for additional information regarding copyright + * ownership. You may not use this file except in compliance with the License. + * + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Copyright 2015 Cloudius Systems + */ + +#include "file_handler.hh" +#include +#include +#include "core/reactor.hh" +#include "core/fstream.hh" +#include "core/shared_ptr.hh" +#include "core/app-template.hh" +#include "exception.hh" + +namespace httpd { + +directory_handler::directory_handler(const sstring& doc_root, + file_transformer* transformer) + : file_interaction_handler(transformer), doc_root(doc_root) { +} + +future> directory_handler::handle(const sstring& path, + std::unique_ptr req, std::unique_ptr rep) { + sstring full_path = doc_root + req->param["path"]; + auto h = this; + return engine().file_type(full_path).then( + [h, full_path, req = std::move(req), rep = std::move(rep)](auto val) mutable { + if (val) { + if (val.value() == directory_entry_type::directory) { + if (h->redirect_if_needed(*req.get(), *rep.get())) { + return make_ready_future>(std::move(rep)); + } + full_path += "/index.html"; + } + return h->read(full_path, std::move(req), std::move(rep)); + } + rep->set_status(reply::status_type::not_found).done(); + return make_ready_future>(std::move(rep)); + + }); +} + +file_interaction_handler::~file_interaction_handler() { + delete transformer; +} + +sstring file_interaction_handler::get_extension(const sstring& file) { + size_t last_slash_pos = file.find_last_of('/'); + size_t last_dot_pos = file.find_last_of('.'); + sstring extension; + if (last_dot_pos != sstring::npos && last_dot_pos > last_slash_pos) { + extension = file.substr(last_dot_pos + 1); + } + return extension; +} + +struct reader { + reader(file f, std::unique_ptr rep) + : is( + make_file_input_stream(make_lw_shared(std::move(f)), + 0, 4096)), _rep(std::move(rep)) { + } + input_stream is; + std::unique_ptr _rep; + + // for input_stream::consume(): + template + void operator()(temporary_buffer data, Done&& done) { + if (data.empty()) { + done(std::move(data)); + _rep->done(); + } else { + _rep->_content.append(data.get(), data.size()); + } + } +}; + +future> file_interaction_handler::read( + const sstring& file_name, std::unique_ptr req, + std::unique_ptr rep) { + sstring extension = get_extension(file_name); + rep->set_content_type(extension); + return engine().open_file_dma(file_name, open_flags::ro).then( + [rep = std::move(rep)](file f) mutable { + std::shared_ptr r = std::make_shared(std::move(f), std::move(rep)); + + return r->is.consume(*r).then([r]() { + r->_rep->done(); + return make_ready_future>(std::move(r->_rep)); + }); + }); +} + +bool file_interaction_handler::redirect_if_needed(const request& req, + reply& rep) const { + if (req._url.length() == 0 || req._url.back() != '/') { + rep.set_status(reply::status_type::moved_permanently); + rep._headers["Location"] = req.get_url() + "/"; + rep.done(); + return true; + } + return false; +} + +future> file_handler::handle(const sstring& path, + std::unique_ptr req, std::unique_ptr rep) { + if (force_path && redirect_if_needed(*req.get(), *rep.get())) { + return make_ready_future>(std::move(rep)); + } + return read(file, std::move(req), std::move(rep)); +} + +} diff --git a/http/file_handler.hh b/http/file_handler.hh new file mode 100644 index 0000000000..2c2bb66c93 --- /dev/null +++ b/http/file_handler.hh @@ -0,0 +1,161 @@ +/* + * This file is open source software, licensed to you under the terms + * of the Apache License, Version 2.0 (the "License"). See the NOTICE file + * distributed with this work for additional information regarding copyright + * ownership. You may not use this file except in compliance with the License. + * + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Copyright 2015 Cloudius Systems + */ + +#ifndef HTTP_FILE_HANDLER_HH_ +#define HTTP_FILE_HANDLER_HH_ + +#include "handlers.hh" + +namespace httpd { +/** + * This is a base class for file transformer. + * + * File transformer adds the ability to modify a file content before returning + * the results. + * + * The transformer decides according to the file extension if transforming is + * needed. + */ +class file_transformer { +public: + /** + * Any file transformer should implement this method. + * @param content the content to transform + * @param req the request + * @param extension the file extension originating the content + */ + virtual void transform(sstring& content, const request& req, + const sstring& extension) = 0; + + virtual ~file_transformer() = default; +}; + +/** + * A base class for handlers that interact with files. + * directory and file handlers both share some common logic + * with regards to file handling. + * they both needs to read a file from the disk, optionally transform it, + * and return the result or page not found on error + */ +class file_interaction_handler : public handler_base { +public: + file_interaction_handler(file_transformer* p = nullptr) + : transformer(p) { + + } + + ~file_interaction_handler(); + + /** + * Allows setting a transformer to be used with the files returned. + * @param t the file transformer to use + * @return this + */ + file_interaction_handler* set_transformer(file_transformer* t) { + transformer = t; + return this; + } + + /** + * if the url ends without a slash redirect + * @param req the request + * @param rep the reply + * @return true on redirect + */ + bool redirect_if_needed(const request& req, reply& rep) const; + + /** + * A helper method that returns the file extension. + * @param file the file to check + * @return the file extension + */ + static sstring get_extension(const sstring& file); + +protected: + + /** + * read a file from the disk and return it in the replay. + * @param file the full path to a file on the disk + * @param req the reuest + * @param rep the reply + */ + future > read(const sstring& file, + std::unique_ptr req, std::unique_ptr rep); + file_transformer* transformer; +}; + +/** + * The directory handler get a disk path in the + * constructor. + * and expect a path parameter in the handle method. + * it would concatenate the two and return the file + * e.g. if the path is /usr/mgmt/public in the path + * parameter is index.html + * handle will return the content of /usr/mgmt/public/index.html + */ +class directory_handler : public file_interaction_handler { +public: + + /** + * The directory handler map a base path and a path parameter to a file + * @param doc_root the root directory to search the file from. + * For example if the root is '/usr/mgmt/public' and the path parameter + * will be '/css/style.css' the file wil be /usr/mgmt/public/css/style.css' + */ + explicit directory_handler(const sstring& doc_root, + file_transformer* transformer = nullptr); + + future> handle(const sstring& path, + std::unique_ptr req, std::unique_ptr rep) override; + +private: + sstring doc_root; +}; + +/** + * The file handler get a path to a file on the disk + * in the constructor. + * it will always return the content of the file. + */ +class file_handler : public file_interaction_handler { +public: + + /** + * The file handler map a file to a url + * @param file the full path to the file on the disk + */ + explicit file_handler(const sstring& file, file_transformer* transformer = + nullptr, bool force_path = true) + : file_interaction_handler(transformer), file(file), force_path( + force_path) { + } + + future> handle(const sstring& path, + std::unique_ptr req, std::unique_ptr rep) override; + +private: + sstring file; + bool force_path; +}; + +} + +#endif /* HTTP_FILE_HANDLER_HH_ */ From f6d253b713f80359406ba2b543fbf51a775e2e9c Mon Sep 17 00:00:00 2001 From: Amnon Heiman Date: Thu, 19 Mar 2015 12:10:02 +0200 Subject: [PATCH 08/12] request return http as the protocol name --- http/request.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http/request.hh b/http/request.hh index 01bad5572f..2c3417f915 100644 --- a/http/request.hh +++ b/http/request.hh @@ -93,7 +93,7 @@ struct request { * Get the request protocol name. Can be either "http" or "https". */ sstring get_protocol_name() const { - return protocol_name; + return "http"; } /** From f2147e74903d87317763cffa8aa44f265ad2d54f Mon Sep 17 00:00:00 2001 From: Amnon Heiman Date: Thu, 19 Mar 2015 12:12:00 +0200 Subject: [PATCH 09/12] Http adding the json path Json path are used when parsing the swagger files. Each path represent an operation in the swagger files. They are used to simplified setting a handler or a function to a path. For example: the code generation would define a json_path like: path_description hello_world("/hello/world",GET,"hello_world", {},{}); Now to define the handler that would use hello_world, you can simply do: hello_world.set(r, [](const_req req) { return "hello world"; }); When r is a reference to a route object. Signed-off-by: Amnon Heiman --- http/json_path.cc | 65 ++++++++++++++++++++++ http/json_path.hh | 135 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 200 insertions(+) create mode 100644 http/json_path.cc create mode 100644 http/json_path.hh diff --git a/http/json_path.cc b/http/json_path.cc new file mode 100644 index 0000000000..b76643fcfd --- /dev/null +++ b/http/json_path.cc @@ -0,0 +1,65 @@ +/* + * This file is open source software, licensed to you under the terms + * of the Apache License, Version 2.0 (the "License"). See the NOTICE file + * distributed with this work for additional information regarding copyright + * ownership. You may not use this file except in compliance with the License. + * + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Copyright 2015 Cloudius Systems + */ + +#include "json_path.hh" + +namespace httpd { + +using namespace std; + +void path_description::set(routes& _routes, handler_base* handler) const { + for (auto& i : mandatory_queryparams) { + handler->mandatory(i); + } + + if (params.size() == 0) + _routes.put(operations.method, path, handler); + else { + match_rule* rule = new match_rule(handler); + rule->add_str(path); + for (auto i = params.begin(); i != params.end(); ++i) { + rule->add_param(std::get<0>(*i), std::get<1>(*i)); + } + _routes.add(rule, operations.method); + } +} + +void path_description::set(routes& _routes, + const json_request_function& f) const { + set(_routes, new function_handler(f)); +} + +path_description::path_description(const sstring& path, operation_type method, + const sstring& nickname, + const std::vector>& path_parameters, + const std::vector& mandatory_params) + : path(path), operations(method, nickname) { + + for (auto man : mandatory_params) { + pushmandatory_param(man); + } + for (auto param : path_parameters) { + params.push_back(param); + } + +} + +} diff --git a/http/json_path.hh b/http/json_path.hh new file mode 100644 index 0000000000..a95eb7b85c --- /dev/null +++ b/http/json_path.hh @@ -0,0 +1,135 @@ +/* + * This file is open source software, licensed to you under the terms + * of the Apache License, Version 2.0 (the "License"). See the NOTICE file + * distributed with this work for additional information regarding copyright + * ownership. You may not use this file except in compliance with the License. + * + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Copyright 2015 Cloudius Systems + */ + +#ifndef JSON_PATH_HH_ +#define JSON_PATH_HH_ + +#include +#include +#include +#include "common.hh" +#include "core/sstring.hh" +#include "routes.hh" +#include "function_handlers.hh" + +namespace httpd { + +/** + * A json_operation contain a method and a nickname. + * operation are associated to a path, that can + * have multiple methods + */ +struct json_operation { + /** + * default constructor + */ + json_operation() + : method(GET) { + } + + /** + * Construct with assignment + * @param method the http method type + * @param nickname the http nickname + */ + json_operation(operation_type method, const sstring& nickname) + : method(method), nickname(nickname) { + } + + operation_type method; + sstring nickname; + +}; + +/** + * path description holds the path in the system. + * It maps a nickname to an operation, which allows + * defining the operation (path and method) by its + * nickname. + * + * the description are taken from the json swagger + * definition file, during auto code generation in the + * compilation. + */ +struct path_description { + /** + * default empty constructor + */ + path_description() = default; + + /** + * constructor for path with parameters + * The constructor is used by + * @param path the url path + * @param method the http method + * @param nickname the nickname + */ + path_description(const sstring& path, operation_type method, + const sstring& nickname, + const std::vector>& path_parameters, + const std::vector& mandatory_params); + + /** + * Add a parameter to the path definition + * for example, if the url should match /file/{path} + * The constructor would be followed by a call to + * pushparam("path") + * + * @param param the name of the parameters, this name will + * be used by the handler to identify the parameters. + * A name can appear at most once in a description + * @param all_path when set to true the parameter will assume to match + * until the end of the url. + * This is useful for situation like file path with + * a rule like /file/{path} and a url /file/etc/hosts. + * path should be equal to /ets/hosts and not only /etc + * @return the current path description + */ + path_description* pushparam(const sstring& param, + bool all_path = false) { + params.push_back( { param, all_path }); + return this; + } + + /** + * adds a mandatory query parameter to the path + * this parameter will be check before calling a handler + * @param param the parameter to head + * @return a pointer to the current path description + */ + path_description* pushmandatory_param(const sstring& param) { + mandatory_queryparams.push_back(param); + return this; + } + + std::vector> params; + sstring path; + json_operation operations; + + std::vector mandatory_queryparams; + + void set(routes& _routes, handler_base* handler) const; + + void set(routes& _routes, const json_request_function& f) const; +}; + +} +#endif /* JSON_PATH_HH_ */ From b93dcf3dbd3d7269b717c211b70c364c69dd76e8 Mon Sep 17 00:00:00 2001 From: Amnon Heiman Date: Thu, 19 Mar 2015 12:24:01 +0200 Subject: [PATCH 10/12] json adding the swagger code generation This takes the json2code perl script from the osv and pass it with small adaptation. It takes a swagger definition file and creates a hh file from it with the following code support. Api opperations are translated to path_description with a name that determine by their nick name. Moduls are defined as json object with the same name. Enums are defined as enum class, a string to enum function is defined for any query enum parameters. For enums that are part of a json object a conversion function is defined so enum of a different type can be assigned to the target enum as long as it has the same enum values. Signed-off-by: Amnon Heiman --- json/json2code.py | 410 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 410 insertions(+) create mode 100755 json/json2code.py diff --git a/json/json2code.py b/json/json2code.py new file mode 100755 index 0000000000..0efee586ce --- /dev/null +++ b/json/json2code.py @@ -0,0 +1,410 @@ +#!/usr/bin/env python +import json +import sys +import re +import glob +import argparse +import os + +parser = argparse.ArgumentParser(description="""Generate C++ class for json +handling from swagger definition""") + +parser.add_argument('--outdir', help='the output directory', default='autogen') +parser.add_argument('-o', help='Output file', default='') +parser.add_argument('-f', help='input file', default='api-java.json') +parser.add_argument('-ns', help="""namespace when set struct will be created +under the namespace""", default='') +parser.add_argument('-jsoninc', help='relative path to the jsaon include', + default='json/') +parser.add_argument('-jsonns', help='set the json namespace', default='json') +parser.add_argument('-indir', help="""when set all json file in the given +directory will be parsed, do not use with -f""", default='') +parser.add_argument('-debug', help='debug level 0 -quite,1-error,2-verbose', + default='1', type=int) +parser.add_argument('-combined', help='set the name of the combined file', + default='autogen/pathautogen.ee') +config = parser.parse_args() + + +valid_vars = {'string': 'sstring', 'int': 'int', 'double': 'double', + 'float': 'float', 'long': 'long', 'boolean': 'bool', 'char': 'char', + 'datetime': 'json::date_time'} + +current_file = '' + +spacing = " " +def getitem(d, key, name): + if key in d: + return d[key] + else: + raise Exception("'" + key + "' not found in " + name) + +def fprint(f, *args): + for arg in args: + f.write(arg) + +def fprintln(f, *args): + for arg in args: + f.write(arg) + f.write('\n') + + +def open_namespace(f, ns=config.ns): + fprintln(f, "namespace ", ns , ' {\n') + + +def close_namespace(f): + fprintln(f, '}') + + +def add_include(f, includes): + for include in includes: + fprintln(f, '#include ', include) + fprintln(f, "") + +def trace_verbose(*params): + if config.debug > 1: + print(''.join(params)) + + +def trace_err(*params): + if config.debug > 0: + print(current_file + ':' + ''.join(params)) + + +def valid_type(param): + if param in valid_vars: + return valid_vars[param] + trace_err("Type [", param, "] not defined") + return param + + +def type_change(param, member): + if param == "array": + if "items" not in member: + trace_err("array without item declaration in ", param) + return "" + item = member["items"] + if "type" in item: + t = item["type"] + elif "$ref" in item: + t = item["$ref"] + else: + trace_err("array items with no type or ref declaration ", param) + return "" + return "json_list< " + valid_type(t) + " >" + return "json_element< " + valid_type(param) + " >" + + + +def print_ind_comment(f, ind, *params): + fprintln(f, ind, "/**") + for s in params: + fprintln(f, ind, " * ", s) + fprintln(f, ind, " */") + +def print_comment(f, *params): + print_ind_comment(f, spacing, *params) + +def print_copyrights(f): + fprintln(f, "/*") + fprintln(f, "* Copyright (C) 2014 Cloudius Systems, Ltd.") + fprintln(f, "*") + fprintln(f, "* This work is open source software, licensed under the", + " terms of the") + fprintln(f, "* BSD license as described in the LICENSE f in the top-", + "level directory.") + fprintln(f, "*") + fprintln(f, "* This is an Auto-Generated-code ") + fprintln(f, "* Changes you do in this file will be erased on next", + " code generation") + fprintln(f, "*/\n") + + +def print_h_file_headers(f, name): + print_copyrights(f) + fprintln(f, "#ifndef __JSON_AUTO_GENERATED_" + name) + fprintln(f, "#define __JSON_AUTO_GENERATED_" + name + "\n") + + +def clean_param(param): + match = re.match(r"(^[^\}]+)\s*}", param) + if match: + return match.group(1) + return param + + +def get_parameter_by_name(obj, name): + for p in obj["parameters"]: + if p["name"] == name: + return p + trace_err ("No Parameter declaration found for ", name) + + +def clear_path_ending(path): + if not path or path[-1] != '/': + return path + return path[0:-1] + + +def add_path(f, path, details): + if "summary" in details: + print_comment(f, details["summary"]) + + if "{" in path: + vals = path.split("{") + vals.reverse() + fprintln(f, spacing, 'path_description::add_path("', clear_path_ending(vals.pop()), + '",', details["method"], ',"', details["nickname"], '")') + while vals: + param = clean_param(vals.pop()) + param_type = get_parameter_by_name(details, param) + if ("allowMultiple" in param_type and + param_type["allowMultiple"] == True): + fprintln(f, spacing, ' ->pushparam("', param, '",true)') + else: + fprintln(f, spacing, ' ->pushparam("', param, '")') + else: + fprintln(f, spacing, 'path_description::add_path("', clear_path_ending(path), '",', + details["method"], ',"', details["nickname"], '")') + if "parameters" in details: + for param in details["parameters"]: + if "required" in param and param["required"] and param["paramType"] == "query": + fprintln(f, spacing, ' ->pushmandatory_param("', param["name"], '")') + fprintln(f, spacing, ";") + + +def get_base_name(param): + return os.path.basename(param) + + +def is_model_valid(name, model): + if name in valid_vars: + return "" + properties = getitem(model[name], "properties", name) + for var in properties: + type = getitem(properties[var], "type", name + ":" + var) + if type == "array": + type = getitem(getitem(properties[var], "items", name + ":" + var), "type", name + ":" + var + ":items") + if type not in valid_vars: + if type not in model: + raise Exception("Unknown type '" + type + "' in Model '" + name + "'") + return type + valid_vars[name] = name + return "" + +def resolve_model_order(data): + res = [] + models = set() + for model_name in data: + visited = set(model_name) + missing = is_model_valid(model_name, data) + resolved = missing == '' + if not resolved: + stack = [model_name] + while not resolved: + if missing in visited: + raise Exception("Cyclic dependency found: " + missing) + missing_depends = is_model_valid(missing, data) + if missing_depends == '': + if missing not in models: + res.append(missing) + models.add(missing) + resolved = len(stack) == 0 + if not resolved: + missing = stack.pop() + else: + stack.append(missing) + missing = missing_depends + elif model_name not in models: + res.append(model_name) + models.add(model_name) + return res + +def create_h_file(data, hfile_name, api_name, init_method, base_api): + if config.o != '': + hfile = open(config.o, "w") + else: + hfile = open(config.outdir + "/" + hfile_name, "w") + print_h_file_headers(hfile, api_name) + add_include(hfile, ['"core/sstring.hh"', '"' + config.jsoninc + + 'json_elements.hh"', '"http/json_path.hh"']) + + add_include(hfile, ['']) + open_namespace(hfile, "httpd") + open_namespace(hfile, api_name) + + if "models" in data: + models_order = resolve_model_order(data["models"]) + for model_name in models_order: + model = data["models"][model_name] + if 'description' in model: + print_ind_comment(hfile, "", model["description"]) + fprintln(hfile, "struct ", model_name, " : public json::json_base {") + member_init = '' + member_assignment = '' + member_copy = '' + for member_name in model["properties"]: + member = model["properties"][member_name] + if "description" in member: + print_comment(hfile, member["description"]) + if "enum" in member: + enum_name = model_name + "_" + member_name + fprintln(hfile, " enum class ", enum_name, " {") + for enum_entry in member["enum"]: + fprintln(hfile, " ", enum_entry, ", ") + fprintln(hfile, "NUM_ITEMS};") + wrapper = member_name + "_wrapper" + fprintln(hfile, " struct ", wrapper, " : public jsonable {") + fprintln(hfile, " ", wrapper, "() = default;") + fprintln(hfile, " virtual std::string to_json() const {") + fprintln(hfile, " switch(v) {") + for enum_entry in member["enum"]: + fprintln(hfile, " case ", enum_name, "::", enum_entry, ": return \"\\\"", enum_entry, "\\\"\";") + fprintln(hfile, " default: return \"Unknown\";") + fprintln(hfile, " }") + fprintln(hfile, " }") + fprintln(hfile, " template") + fprintln(hfile, " ", wrapper, "(const T& _v) {") + fprintln(hfile, " switch(_v) {") + for enum_entry in member["enum"]: + fprintln(hfile, " case T::", enum_entry, ": v = ", enum_name, "::", enum_entry, "; break;") + fprintln(hfile, " default: v = ", enum_name, "::NUM_ITEMS;") + fprintln(hfile, " }") + fprintln(hfile, " }") + fprintln(hfile, " ", enum_name, " v;") + fprintln(hfile, " };") + fprintln(hfile, " ", config.jsonns, "::json_element<", + member_name, "_wrapper> ", + member_name, ";\n") + else: + fprintln(hfile, " ", config.jsonns, "::", + type_change(member["type"], member), " ", + member_name, ";\n") + member_init += " add(&" + member_name + ',"' + member_init += member_name + '");\n' + member_assignment += " " + member_name + " = " + "e." + member_name + ";\n" + member_copy += " e." + member_name + " = " + member_name + ";\n" + fprintln(hfile, "void register_params() {") + fprintln(hfile, member_init) + fprintln(hfile, '}') + + fprintln(hfile, model_name, '() {') + fprintln(hfile, ' register_params();') + fprintln(hfile, '}') + fprintln(hfile, model_name, '(const ' + model_name + ' & e) {') + fprintln(hfile, ' register_params();') + fprintln(hfile, member_assignment) + fprintln(hfile, '}') + fprintln(hfile, "template") + fprintln(hfile, model_name, "& operator=(const ", "T& e) {") + fprintln(hfile, member_assignment) + fprintln(hfile, " return *this;") + fprintln(hfile, "}") + fprintln(hfile, model_name, "& operator=(const ", model_name, "& e) {") + fprintln(hfile, member_assignment) + fprintln(hfile, " return *this;") + fprintln(hfile, "}") + fprintln(hfile, "template") + fprintln(hfile, model_name, "& update(T& e) {") + fprintln(hfile, member_copy) + fprintln(hfile, " return *this;") + fprintln(hfile, "}") + fprintln(hfile, "};\n\n") + + # print_ind_comment(hfile, "", "Initialize the path") +# fprintln(hfile, init_method + "(const std::string& description);") + fprintln(hfile, 'static const sstring name = "', base_api, '";') + for item in data["apis"]: + path = item["path"] + if "operations" in item: + for oper in item["operations"]: + if "summary" in oper: + print_comment(hfile, oper["summary"]) + vals = path.split("{") + vals.reverse() + + fprintln(hfile, 'static const path_description ', getitem(oper, "nickname", oper), '("', clear_path_ending(vals.pop()), + '",', oper["method"], ',"', oper["nickname"], '",') + fprint(hfile, '{') + first = True + while vals: + path_param = clean_param(vals.pop()) + path_param_type = get_parameter_by_name(oper, path_param) + if first == True: + first = False + else: + fprint(hfile, "\n,") + if ("allowMultiple" in path_param_type and + path_param_type["allowMultiple"] == True): + fprint(hfile, '{', '"', path_param , '", true', '}') + else: + fprint(hfile, '{', '"', path_param , '", false', '}') + fprint(hfile, '}') + fprint(hfile, ',{') + first = True + if "parameters" in oper: + enum_definitions = "" + for param in oper["parameters"]: + if "required" in param and param["required"] and param["paramType"] == "query": + if first == True: + first = False + else: + fprint(hfile, "\n,") + fprint(hfile, '"', param["name"], '"') + if "enum" in param: + enum_definitions = enum_definitions + 'namespace ns_' + oper["nickname"] + '{\n' + enm = param["name"] + enum_definitions = enum_definitions + 'enum class ' + enm + ' {' + for val in param["enum"]: + enum_definitions = enum_definitions + val + ", " + enum_definitions = enum_definitions + 'NUM_ITEMS};\n' + enum_definitions = enum_definitions + enm + ' str2' + enm + '(const sstring& str) {\n' + enum_definitions = enum_definitions + ' static const sstring arr[] = {"' + '","'.join(param["enum"]) + '"};\n' + enum_definitions = enum_definitions + ' int i;\n' + enum_definitions = enum_definitions + ' for (i=0; i < ' + str(len(param["enum"])) + '; i++) {\n' + enum_definitions = enum_definitions + ' if (arr[i] == str) {return (' + enm + ')i;}\n}\n' + enum_definitions = enum_definitions + ' return (' + enm + ')i;\n' + enum_definitions = enum_definitions + '}\n}\n' + + fprintln(hfile, '});') + fprintln(hfile, enum_definitions) + + close_namespace(hfile) + close_namespace(hfile) + hfile.write("#endif //__JSON_AUTO_GENERATED_HEADERS\n") + hfile.close() + +def parse_file(param, combined): + global current_file + trace_verbose("parsing ", param, " file") + try: + json_data = open(param) + data = json.load(json_data) + json_data.close() + except: + type, value, tb = sys.exc_info() + print("Bad formatted JSON file '" + param + "' error ", value.message) + sys.exit(-1) + try: + base_file_name = get_base_name(param) + current_file = base_file_name + hfile_name = base_file_name + ".hh" + api_name = base_file_name.replace('.', '_') + base_api = base_file_name.replace('.json', '') + init_method = "void " + api_name + "_init_path" + trace_verbose("creating ", hfile_name) + if (combined): + fprintln(combined, '#include "', base_file_name, ".cc", '"') + create_h_file(data, hfile_name, api_name, init_method, base_api) + except: + type, value, tb = sys.exc_info() + print("Error while parsing JSON file '" + param + "' error ", value.message) + sys.exit(-1) + +if "indir" in config and config.indir != '': + combined = open(config.combined, "w") + for f in glob.glob(os.path.join(config.indir, "*.json")): + parse_file(f, combined) +else: + parse_file(config.f, None) From ff6d48bb27de2555b68f18ac2f5ceb7d0fc37ffb Mon Sep 17 00:00:00 2001 From: Amnon Heiman Date: Thu, 19 Mar 2015 12:38:19 +0200 Subject: [PATCH 11/12] adding swagger code generation to configure --- configure.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/configure.py b/configure.py index 939fc06af1..df5877a5c8 100755 --- a/configure.py +++ b/configure.py @@ -321,6 +321,9 @@ with open(buildfile, 'w') as f: rule gen command = echo -e $text > $out description = GEN $out + rule swagger + command = json/json2code.py -f $in -o $out + description = SWAGGER $out ''').format(**globals())) for mode in build_modes: modeval = modes[mode] @@ -347,6 +350,7 @@ with open(buildfile, 'w') as f: artifacts = str.join(' ', ('$builddir/' + mode + '/' + x for x in build_artifacts)))) compiles = {} ragels = {} + swaggers = {} for binary in build_artifacts: srcs = deps[binary] objs = ['$builddir/' + mode + '/' + src.replace('.cc', '.o') @@ -375,15 +379,21 @@ with open(buildfile, 'w') as f: elif src.endswith('.rl'): hh = '$builddir/' + mode + '/gen/' + src.replace('.rl', '.hh') ragels[hh] = src + elif src.endswith('.json'): + hh = '$builddir/' + mode + '/gen/' + src + '.hh' + swaggers[hh] = src else: raise Exception('No rule for ' + src) for obj in compiles: src = compiles[obj] - gen_headers = ragels.keys() + gen_headers = list(ragels.keys()) + list(swaggers.keys()) f.write('build {}: cxx.{} {} || {} \n'.format(obj, mode, src, ' '.join(gen_headers))) for hh in ragels: src = ragels[hh] f.write('build {}: ragel {}\n'.format(hh, src)) + for hh in swaggers: + src = swaggers[hh] + f.write('build {}: swagger {}\n'.format(hh,src)) f.write(textwrap.dedent('''\ rule configure command = python3 configure.py $configure_args From 006b061aba37a8cdb4cca202d256fe0f14c973c0 Mon Sep 17 00:00:00 2001 From: Amnon Heiman Date: Thu, 19 Mar 2015 12:34:53 +0200 Subject: [PATCH 12/12] Adding hello world swagger demo This demonstrate how to use a swagger definition file to create an API. The swagger file demo.json define one api called hello_world, it has both query and path parameters and return an object as a result. The handler implementation simply places the given parameters in the return object. Signed-off-by: Amnon Heiman --- apps/httpd/demo.json | 73 ++++++++++++++++++++++++++++++++++++++++++++ apps/httpd/main.cc | 11 +++++++ configure.py | 4 +-- 3 files changed, 86 insertions(+), 2 deletions(-) create mode 100644 apps/httpd/demo.json diff --git a/apps/httpd/demo.json b/apps/httpd/demo.json new file mode 100644 index 0000000000..12261c453f --- /dev/null +++ b/apps/httpd/demo.json @@ -0,0 +1,73 @@ +{ + "apiVersion": "0.0.1", + "swaggerVersion": "1.2", + "basePath": "{{Protocol}}://{{Host}}", + "resourcePath": "/hello", + "produces": [ + "application/json" + ], + "apis": [ + { + "path": "/hello/world/{var1}/{var2}", + "operations": [ + { + "method": "GET", + "summary": "Returns the number of seconds since the system was booted", + "type": "long", + "nickname": "hello_world", + "produces": [ + "application/json" + ], + "parameters": [ + { + "name":"var2", + "description":"Full path of file or directory", + "required":true, + "allowMultiple":true, + "type":"string", + "paramType":"path" + }, + { + "name":"var1", + "description":"Full path of file or directory", + "required":true, + "allowMultiple":false, + "type":"string", + "paramType":"path" + }, + { + "name":"query_enum", + "description":"The operation to perform", + "required":true, + "allowMultiple":false, + "type":"string", + "paramType":"query", + "enum":["VAL1", "VAL2", "VAL3"] + } + ] + } + ] + } + ], + "models" : { + "my_object": { + "id": "my_object", + "description": "Demonstrate an object", + "properties": { + "var1": { + "type": "string", + "description": "The first parameter in the path" + }, + "var2": { + "type": "string", + "description": "The second parameter in the path" + }, + "enum_var" : { + "type": "string", + "description": "Demonstrate an enum returned, note this is not the same enum type of the request", + "enum":["VAL1", "VAL2", "VAL3"] + } + } + } + } +} diff --git a/apps/httpd/main.cc b/apps/httpd/main.cc index 0523a1b897..e9502c816c 100644 --- a/apps/httpd/main.cc +++ b/apps/httpd/main.cc @@ -23,6 +23,7 @@ #include "http/handlers.hh" #include "http/function_handlers.hh" #include "http/file_handler.hh" +#include "apps/httpd/demo.json.hh" namespace bpo = boost::program_options; @@ -45,6 +46,16 @@ void set_routes(routes& r) { r.add(operation_type::GET, url("/"), h1); r.add(operation_type::GET, url("/file").remainder("path"), new directory_handler("/")); + demo_json::hello_world.set(r, + [](const_req req) { + demo_json::my_object obj; + obj.var1 = req.param.at("var1"); + obj.var2 = req.param.at("var2"); + demo_json::ns_hello_world::query_enum v = demo_json::ns_hello_world::str2query_enum(req.query_parameters.at("query_enum")); + // This demonstrate enum conversion + obj.enum_var = v; + return obj; + }); } int main(int ac, char** av) { diff --git a/configure.py b/configure.py index df5877a5c8..1ef7e9f538 100755 --- a/configure.py +++ b/configure.py @@ -210,7 +210,7 @@ deps = { 'seastar.pc': [], 'apps/seastar/seastar': ['apps/seastar/main.cc'] + core, 'tests/test-reactor': ['tests/test-reactor.cc'] + core, - 'apps/httpd/httpd': ['http/common.cc', 'http/routes.cc', 'json/json_elements.cc', 'json/formatter.cc', 'http/matcher.cc', 'http/mime_types.cc', 'http/httpd.cc', 'http/reply.cc', 'http/request_parser.rl', 'apps/httpd/main.cc'] + libnet + core, + 'apps/httpd/httpd': ['apps/httpd/demo.json', 'http/json_path.cc', 'http/file_handler.cc', 'http/common.cc', 'http/routes.cc', 'json/json_elements.cc', 'json/formatter.cc', 'http/matcher.cc', 'http/mime_types.cc', 'http/httpd.cc', 'http/reply.cc', 'http/request_parser.rl', 'apps/httpd/main.cc'] + libnet + core, 'apps/memcached/memcached': ['apps/memcached/memcache.cc'] + memcache_base, 'tests/memcached/test_ascii_parser': ['tests/memcached/test_ascii_parser.cc'] + memcache_base, 'tests/fileiotest': ['tests/fileiotest.cc'] + core, @@ -230,7 +230,7 @@ deps = { 'apps/seawreck/seawreck': ['apps/seawreck/seawreck.cc', 'apps/seawreck/http_response_parser.rl'] + core + libnet, 'tests/blkdiscard_test': ['tests/blkdiscard_test.cc'] + core, 'tests/sstring_test': ['tests/sstring_test.cc'] + core, - 'tests/httpd': ['http/common.cc', 'http/routes.cc', 'json/json_elements.cc', 'json/formatter.cc', 'http/matcher.cc', 'tests/httpd.cc', 'http/mime_types.cc', 'http/reply.cc'] + core, + 'tests/httpd': ['http/common.cc', 'http/routes.cc', 'http/json_path.cc', 'json/json_elements.cc', 'json/formatter.cc', 'http/matcher.cc', 'tests/httpd.cc', 'http/mime_types.cc', 'http/reply.cc'] + core, 'tests/allocator_test': ['tests/allocator_test.cc', 'core/memory.cc', 'core/posix.cc'], 'tests/output_stream_test': ['tests/output_stream_test.cc'] + core + libnet, 'tests/udp_zero_copy': ['tests/udp_zero_copy.cc'] + core + libnet,