Merge pull request #8911 from kpfleming/add-api-docs-endpoint

Add '/api/docs' endpoint to Auth server

.github/actions/spell-check/expect.txt

@@ -69,6 +69,7 @@ ANSSI
 Antoin
 anycast
 api
+apidocfiles
 apikey
 APIv
 AQAB

.github/workflows/codeql-analysis.yml

@@ -33,6 +33,12 @@ jobs:
     - run: git checkout HEAD^2
       if: ${{ github.event_name == 'pull_request' }}
 
+    # Python and virtualenv are required for building the Authoritative server
+    - uses: actions/setup-python@v2
+      with:
+        python-version: '2.7'
+    - run: pip install virtualenv
+
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
       uses: github/codeql-action/init@v1

docs/http-api/index.rst

@@ -318,7 +318,9 @@ Working with the API
 
 This chapter describes the PowerDNS Authoritative API.
 When creating an API wrapper (for instance when fronting multiple API's), it is recommended to stick to this API specification.
-The API is described in the `OpenAPI format <https://www.openapis.org/>`_, also known as "Swagger", and this description is `available <https://raw.githubusercontent.com/PowerDNS/pdns/master/docs/http-api/swagger/authoritative-api-swagger.yaml>`_.
+The API is described in the `OpenAPI format <https://www.openapis.org/>`_, also known as "Swagger", and this description is `available <https://raw.githubusercontent.com/PowerDNS/pdns/master/docs/http-api/swagger/authoritative-api-swagger.yaml>`_. It can also be obtained from a running server if the administrator of that server has enabled the API; it
+is available at the  `/api/docs` endpoint in both YAML and JSON formats (the 'Accept' header can be used to indicate the
+desired format).
 
 Authentication
 ~~~~~~~~~~~~~~

docs/http-api/swagger/authoritative-api-swagger.yaml

@@ -4,10 +4,7 @@ info:
   title: PowerDNS Authoritative HTTP API
   license:
     name: MIT
-host: localhost:8081
 basePath: /api/v1
-schemes:
-  - http
 consumes:
   - application/json
 produces:
@@ -406,9 +403,9 @@ paths:
           schema:
             type: array
             items:
-            - $ref: '#/definitions/StatisticItem'
-            - $ref: '#/definitions/MapStatisticItem'
-            - $ref: '#/definitions/RingStatisticItem'
+              - $ref: '#/definitions/StatisticItem'
+              - $ref: '#/definitions/MapStatisticItem'
+              - $ref: '#/definitions/RingStatisticItem'
         '422':
           description: 'Returned when a non-existing statistic name has been requested. Contains an error message'
 

ext/Makefile.am

@@ -9,4 +9,5 @@ DIST_SUBDIRS = \
 	yahttp
 
 EXTRA_DIST = \
-	luawrapper/include/LuaContext.hpp
+	luawrapper/include/LuaContext.hpp \
+	incbin/incbin.h

lgtm.yml

@@ -23,6 +23,9 @@ extraction:
       - "sed -i 's/codedocs docs//' Makefile.am"
       - "autoreconf -vi"
       - "./configure --with-modules='bind gsqlite3 gmysql gpgsql ldap'"
+      - "pushd pdns"
+      - "touch .venv api-swagger.yaml api-swagger.json"
+      - "popd"
     index:
       build_command:
       - "pushd pdns/dnsdistdist"

m4/pdns_check_virtualenv.m4

@@ -8,5 +8,5 @@ AC_DEFUN([PDNS_CHECK_VIRTUALENV], [
   ])
   AM_CONDITIONAL([HAVE_VIRTUALENV], [test "x$VIRTUALENV" != "xno"])
   AM_CONDITIONAL([HAVE_MANPAGES], [test -e "$srcdir/docs/pdns_server.1"])
+  AM_CONDITIONAL([HAVE_API_SWAGGER_JSON], [test -e "$srcdir/pdns/api-swagger.json"])
 ])
-

pdns/.gitignore

@@ -55,7 +55,10 @@ version_generated.h
 /pubsuffix.cc
 /calidns
 /dumresp
-/htmlfiles.h
+/apidocfiles.h
+/api-swagger.yaml
+/api-swagger.json
+/.venv
 effective_tld_names.dat
 /dnsmessage.pb.cc
 /dnsmessage.pb.h

pdns/Makefile.am

@@ -50,12 +50,17 @@ EXTRA_DIST = \
 	ixfrdist.example.yml \
 	lua-record.cc \
 	minicurl.cc \
-	minicurl.hh
+	minicurl.hh \
+	api-swagger.yaml \
+	api-swagger.json \
+	requirements.txt \
+	incfiles
 
 BUILT_SOURCES = \
 	bind-dnssec.schema.sqlite3.sql.h \
 	bindparser.h \
-	dnslabeltext.cc
+	dnslabeltext.cc \
+	apidocfiles.h
 
 CLEANFILES = \
 	*.gcda \
@@ -65,7 +70,34 @@ CLEANFILES = \
 	backends/gsql/gsqlbackend.gcno \
 	backends/gsql/gsqlbackend.gcov \
 	dnsmessage.pb.cc dnsmessage.pb.h \
-	pdns.conf-dist
+	pdns.conf-dist \
+	apidocfiles.h \
+	api-swagger.yaml \
+	api-swagger.json
+
+# use a $(wildcard) wrapper here to allow build to proceed if output
+# file is present but input file is not (e.g. in a dist tarball)
+api-swagger.yaml: $(wildcard ../docs/http-api/swagger/authoritative-api-swagger.yaml)
+	cp $< $@
+
+if HAVE_VIRTUALENV
+.venv: requirements.txt
+	virtualenv .venv
+	.venv/bin/pip install -U pip setuptools setuptools-git
+	.venv/bin/pip install -r requirements.txt
+
+api-swagger.json: api-swagger.yaml .venv
+	.venv/bin/python -c "import sys, json, yaml; y = yaml.safe_load(sys.stdin.read()); json.dump(y, sys.stdout, indent=2, separators=(',', ': '))" < $< > $@
+else # if HAVE_VIRTUALENV
+if !HAVE_API_SWAGGER_JSON
+api-swagger.json:
+	echo "You need virtualenv to generate the JSON API document"
+	exit 1
+endif
+endif
+
+apidocfiles.h: api-swagger.yaml api-swagger.json
+	./incfiles $^ > $@
 
 noinst_SCRIPTS = pdns.init
 sysconf_DATA = pdns.conf-dist

pdns/incfiles

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+export LC_ALL=C.UTF-8
+export LANG=C.UTF-8
+
+for a in $@
+do
+	c=$(echo $a | tr "/.-" "___")
+	echo "INCBIN(${c}, \"${a}\");"
+	echo "static const string g_${c}{(const char*)g${c}Data, g${c}Size};"
+done

pdns/requirements.txt

@@ -0,0 +1 @@
+PyYAML

pdns/webserver.cc

@@ -81,21 +81,43 @@ bool HttpRequest::compareHeader(const string &header_name, const string &expecte
   return (0==strcmp(header->second.c_str(), expected_value.c_str()));
 }
 
+void HttpResponse::setPlainBody(const string& document)
+{
+  this->headers["Content-Type"] = "text/plain; charset=utf-8";
+
+  this->body = document;
+}
 
-void HttpResponse::setBody(const json11::Json& document)
+void HttpResponse::setYamlBody(const string& document)
 {
+  this->headers["Content-Type"] = "application/x-yaml";
+
+  this->body = document;
+}
+
+void HttpResponse::setJsonBody(const string& document)
+{
+  this->headers["Content-Type"] = "application/json";
+
+  this->body = document;
+}
+
+void HttpResponse::setJsonBody(const json11::Json& document)
+{
+  this->headers["Content-Type"] = "application/json";
+
   document.dump(this->body);
 }
 
 void HttpResponse::setErrorResult(const std::string& message, const int status_)
 {
-  setBody(json11::Json::object { { "error", message } });
+  setJsonBody(json11::Json::object { { "error", message } });
   this->status = status_;
 }
 
 void HttpResponse::setSuccessResult(const std::string& message, const int status_)
 {
-  setBody(json11::Json::object { { "result", message } });
+  setJsonBody(json11::Json::object { { "result", message } });
   this->status = status_;
 }
 
@@ -150,8 +172,6 @@ void WebServer::apiWrapper(WebServer::HandlerFunction handler, HttpRequest* req,
     throw HttpUnauthorizedException("X-API-Key");
   }
 
-  resp->headers["Content-Type"] = "application/json";
-
   // security headers
   resp->headers["X-Content-Type-Options"] = "nosniff";
   resp->headers["X-Frame-Options"] = "deny";
@@ -233,8 +253,12 @@ void WebServer::handleRequest(HttpRequest& req, HttpResponse& resp) const
     YaHTTP::strstr_map_t::iterator header;
 
     if ((header = req.headers.find("accept")) != req.headers.end()) {
-      // json wins over html
-      if (header->second.find("application/json") != std::string::npos) {
+      // yaml wins over json, json wins over html
+      if (header->second.find("application/x-yaml") != std::string::npos) {
+        req.accept_yaml = true;
+      } else if (header->second.find("text/x-yaml") != std::string::npos) {
+        req.accept_yaml = true;
+      } else if (header->second.find("application/json") != std::string::npos) {
         req.accept_json = true;
       } else if (header->second.find("text/html") != std::string::npos) {
         req.accept_html = true;
@@ -272,14 +296,14 @@ void WebServer::handleRequest(HttpRequest& req, HttpResponse& resp) const
     // TODO rm this logline?
     g_log<<Logger::Debug<<req.logprefix<<"Error result for \"" << req.url.path << "\": " << resp.status << endl;
     string what = YaHTTP::Utility::status2text(resp.status);
-    if(req.accept_html) {
-      resp.headers["Content-Type"] = "text/html; charset=utf-8";
-      resp.body = "<!html><title>" + what + "</title><h1>" + what + "</h1>";
-    } else if (req.accept_json) {
+    if (req.accept_json) {
       resp.headers["Content-Type"] = "application/json";
       if (resp.body.empty()) {
         resp.setErrorResult(what, resp.status);
       }
+    } else if (req.accept_html) {
+      resp.headers["Content-Type"] = "text/html; charset=utf-8";
+      resp.body = "<!html><title>" + what + "</title><h1>" + what + "</h1>";
     } else {
       resp.headers["Content-Type"] = "text/plain; charset=utf-8";
       resp.body = what;

pdns/webserver.hh

@@ -33,6 +33,7 @@ class HttpRequest : public YaHTTP::Request {
 public:
   HttpRequest(const string& logprefix_="") : YaHTTP::Request(), accept_json(false), accept_html(false), complete(false), logprefix(logprefix_) { };
 
+  bool accept_yaml;
   bool accept_json;
   bool accept_html;
   bool complete;
@@ -49,7 +50,10 @@ public:
   HttpResponse() : YaHTTP::Response() { };
   HttpResponse(const YaHTTP::Response &resp) : YaHTTP::Response(resp) { };
 
-  void setBody(const json11::Json& document);
+  void setPlainBody(const string& document);
+  void setYamlBody(const string& document);
+  void setJsonBody(const string& document);
+  void setJsonBody(const json11::Json& document);
   void setErrorResult(const std::string& message, const int status);
   void setSuccessResult(const std::string& message, const int status = 200);
 };

pdns/ws-api.cc

@@ -114,22 +114,22 @@ void apiDiscovery(HttpRequest* req, HttpResponse* resp) {
   };
   Json doc = Json::array { version1 };
 
-  resp->setBody(doc);
+  resp->setJsonBody(doc);
 }
 
 void apiServer(HttpRequest* req, HttpResponse* resp) {
   if(req->method != "GET")
     throw HttpMethodNotAllowedException();
 
   Json doc = Json::array {getServerDetail()};
-  resp->setBody(doc);
+  resp->setJsonBody(doc);
 }
 
 void apiServerDetail(HttpRequest* req, HttpResponse* resp) {
   if(req->method != "GET")
     throw HttpMethodNotAllowedException();
 
-  resp->setBody(getServerDetail());
+  resp->setJsonBody(getServerDetail());
 }
 
 void apiServerConfig(HttpRequest* req, HttpResponse* resp) {
@@ -151,7 +151,7 @@ void apiServerConfig(HttpRequest* req, HttpResponse* resp) {
       { "value", value },
     });
   }
-  resp->setBody(doc);
+  resp->setJsonBody(doc);
 }
 
 void apiServerStatistics(HttpRequest* req, HttpResponse* resp) {
@@ -172,7 +172,7 @@ void apiServerStatistics(HttpRequest* req, HttpResponse* resp) {
       { "value", std::to_string(*stat) },
     });
 
-    resp->setBody(doc);
+    resp->setJsonBody(doc);
 
     return;
   }
@@ -273,7 +273,7 @@ void apiServerStatistics(HttpRequest* req, HttpResponse* resp) {
   }
 #endif
 
-  resp->setBody(doc);
+  resp->setJsonBody(doc);
 }
 
 DNSName apiNameToDNSName(const string& name) {