From 236418813e05e4ee3034558115f3be99ff38beb7 Mon Sep 17 00:00:00 2001 From: Justin Clift Date: Thu, 27 Apr 2023 22:00:35 +1000 Subject: [PATCH] api, common. Log all API calls into our database, for stats generation With this commit, we record basic info for each API call so we can start doing useful statistic generation for API usage. The backend database needs a new table created to hold the info: CREATE TABLE public.api_call_log ( api_call_id bigint NOT NULL, api_call_date timestamp with time zone DEFAULT now(), caller_id bigint, db_owner_id bigint, db_id bigint, api_operation text NOT NULL, api_caller_sw text ); COMMENT ON COLUMN public.api_call_log.db_owner_id IS 'This field must be nullable, as not all api calls act on a database'; COMMENT ON COLUMN public.api_call_log.db_id IS 'This field must be nullable, as not all api calls act on a database'; CREATE SEQUENCE public.api_log_log_id_seq START WITH 1 INCREMENT BY 1 NO MINVALUE NO MAXVALUE CACHE 1; ALTER SEQUENCE public.api_log_log_id_seq OWNED BY public.api_call_log.api_call_id; ALTER TABLE ONLY public.api_call_log ALTER COLUMN api_call_id SET DEFAULT nextval('public.api_log_log_id_seq'::regclass); ALTER TABLE ONLY public.api_call_log ADD CONSTRAINT api_log_users_user_id_fk FOREIGN KEY (caller_id) REFERENCES public.users(user_id); --- api/handlers.go | 64 +++++++++++++++++++++++++++++++++++++++----- api/main.go | 54 +++++++++++++++++-------------------- common/postgresql.go | 45 +++++++++++++++++++++++++++++++ database/dbhub.sql | 56 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 183 insertions(+), 36 deletions(-) diff --git a/api/handlers.go b/api/handlers.go index d7cd28310..5488ca5f7 100644 --- a/api/handlers.go +++ b/api/handlers.go @@ -27,12 +27,15 @@ import ( // * "dbname" is the name of the database func branchesHandler(w http.ResponseWriter, r *http.Request) { // Do auth check, grab request info - _, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r) + loggedInUser, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r) if err != nil { jsonErr(w, err.Error(), httpStatus) return } + // Record the api call in our backend database + com.ApiCallLog(loggedInUser, dbOwner, dbName, "branches", r.Header.Get("User-Agent")) + // If the database is a live database, we return an error message isLive, _, err := com.CheckDBLive(dbOwner, dbName) if err != nil { @@ -102,6 +105,9 @@ func columnsHandler(w http.ResponseWriter, r *http.Request) { return } + // Record the api call in our backend database + com.ApiCallLog(loggedInUser, dbOwner, dbName, "columns", r.Header.Get("User-Agent")) + // Extract the table name table, err := com.GetFormTable(r, false) if err != nil { @@ -221,12 +227,15 @@ func columnsHandler(w http.ResponseWriter, r *http.Request) { // * "dbname" is the name of the database func commitsHandler(w http.ResponseWriter, r *http.Request) { // Do auth check, grab request info - _, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r) + loggedInUser, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r) if err != nil { jsonErr(w, err.Error(), httpStatus) return } + // Record the api call in our backend database + com.ApiCallLog(loggedInUser, dbOwner, dbName, "commits", r.Header.Get("User-Agent")) + // If the database is a live database, we return an error message isLive, _, err := com.CheckDBLive(dbOwner, dbName) if err != nil { @@ -271,6 +280,9 @@ func databasesHandler(w http.ResponseWriter, r *http.Request) { return } + // Record the api call in our backend database + com.ApiCallLog(loggedInUser, "", "", "databases", r.Header.Get("User-Agent")) + // Get "live" boolean value, if provided by the caller var live bool live, err = com.GetFormLive(r) @@ -336,6 +348,9 @@ func deleteHandler(w http.ResponseWriter, r *http.Request) { } dbOwner := loggedInUser + // Record the api call in our backend database + com.ApiCallLog(loggedInUser, dbOwner, dbName, "delete", r.Header.Get("User-Agent")) + // Check if the database exists exists, err := com.CheckDBPermissions(loggedInUser, dbOwner, dbName, false) if err != nil { @@ -517,6 +532,10 @@ func diffHandler(w http.ResponseWriter, r *http.Request) { return } + // Record the api call in our backend database + // Note - Lets not bother adding additional api logging fields just for the diff function at this stage + com.ApiCallLog(loggedInUser, dbOwnerA, dbNameA, "diff", r.Header.Get("User-Agent")) + // Check permissions of the first database var allowed bool allowed, err = com.CheckDBPermissions(loggedInUser, dbOwnerA, dbNameA, false) @@ -593,6 +612,9 @@ func downloadHandler(w http.ResponseWriter, r *http.Request) { return } + // Record the api call in our backend database + com.ApiCallLog(loggedInUser, dbOwner, dbName, "download", r.Header.Get("User-Agent")) + // Return the requested database to the user _, err = com.DownloadDatabase(w, r, dbOwner, dbName, commitID, loggedInUser, "api") if err != nil { @@ -630,6 +652,9 @@ func executeHandler(w http.ResponseWriter, r *http.Request) { return } + // Record the api call in our backend database + com.ApiCallLog(loggedInUser, dbOwner, dbName, "execute", r.Header.Get("User-Agent")) + // Grab the incoming SQLite query rawInput := r.FormValue("sql") var sql string @@ -709,6 +734,9 @@ func indexesHandler(w http.ResponseWriter, r *http.Request) { return } + // Record the api call in our backend database + com.ApiCallLog(loggedInUser, dbOwner, dbName, "indexes", r.Header.Get("User-Agent")) + // Check if the database is a live database, and get the node/queue to send the request to isLive, liveNode, err := com.CheckDBLive(dbOwner, dbName) if err != nil { @@ -826,12 +854,15 @@ func indexesHandler(w http.ResponseWriter, r *http.Request) { // * "dbname" is the name of the database func metadataHandler(w http.ResponseWriter, r *http.Request) { // Do auth check, grab request info - _, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r) + loggedInUser, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r) if err != nil { jsonErr(w, err.Error(), httpStatus) return } + // Record the api call in our backend database + com.ApiCallLog(loggedInUser, dbOwner, dbName, "metadata", r.Header.Get("User-Agent")) + // If the database is a live database, we return an error message isLive, _, err := com.CheckDBLive(dbOwner, dbName) if err != nil { @@ -886,6 +917,9 @@ func queryHandler(w http.ResponseWriter, r *http.Request) { return } + // Record the api call in our backend database + com.ApiCallLog(loggedInUser, dbOwner, dbName, "query", r.Header.Get("User-Agent")) + // Grab the incoming SQLite query rawInput := r.FormValue("sql") query, err := com.CheckUnicode(rawInput) @@ -958,12 +992,15 @@ func queryHandler(w http.ResponseWriter, r *http.Request) { // * "dbname" is the name of the database func releasesHandler(w http.ResponseWriter, r *http.Request) { // Do auth check, grab request info - _, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r) + loggedInUser, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r) if err != nil { jsonErr(w, err.Error(), httpStatus) return } + // Record the api call in our backend database + com.ApiCallLog(loggedInUser, dbOwner, dbName, "releases", r.Header.Get("User-Agent")) + // If the database is a live database, we return an error message isLive, _, err := com.CheckDBLive(dbOwner, dbName) if err != nil { @@ -1033,6 +1070,9 @@ func tablesHandler(w http.ResponseWriter, r *http.Request) { return } + // Record the api call in our backend database + com.ApiCallLog(loggedInUser, dbOwner, dbName, "tables", r.Header.Get("User-Agent")) + // Check if the database is a live database, and get the node/queue to send the request to isLive, liveNode, err := com.CheckDBLive(dbOwner, dbName) if err != nil { @@ -1108,12 +1148,15 @@ func tablesHandler(w http.ResponseWriter, r *http.Request) { // * "dbname" is the name of the database func tagsHandler(w http.ResponseWriter, r *http.Request) { // Do auth check, grab request info - _, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r) + loggedInUser, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r) if err != nil { jsonErr(w, err.Error(), httpStatus) return } + // Record the api call in our backend database + com.ApiCallLog(loggedInUser, dbOwner, dbName, "tags", r.Header.Get("User-Agent")) + // If the database is a live database, we return an error message isLive, _, err := com.CheckDBLive(dbOwner, dbName) if err != nil { @@ -1263,6 +1306,9 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) { return } + // Record the api call in our backend database + com.ApiCallLog(loggedInUser, loggedInUser, dbName, "upload", r.Header.Get("User-Agent")) + // Check if the database exists already exists, err := com.CheckDBExists(loggedInUser, dbName) if err != nil { @@ -1366,6 +1412,9 @@ func viewsHandler(w http.ResponseWriter, r *http.Request) { return } + // Record the api call in our backend database + com.ApiCallLog(loggedInUser, loggedInUser, dbName, "views", r.Header.Get("User-Agent")) + // Check if the database is a live database, and get the node/queue to send the request to isLive, liveNode, err := com.CheckDBLive(dbOwner, dbName) if err != nil { @@ -1441,12 +1490,15 @@ func viewsHandler(w http.ResponseWriter, r *http.Request) { // * "dbname" is the name of the database being queried func webpageHandler(w http.ResponseWriter, r *http.Request) { // Authenticate user and collect requested database details - _, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r) + loggedInUser, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r) if err != nil { jsonErr(w, err.Error(), httpStatus) return } + // Record the api call in our backend database + com.ApiCallLog(loggedInUser, dbOwner, dbName, "views", r.Header.Get("User-Agent")) + // Return the database webUI URL to the user var z com.WebpageResponseContainer z.WebPage = "https://" + com.Conf.Web.ServerName + "/" + dbOwner + "/" + dbName diff --git a/api/main.go b/api/main.go index 11c305669..d7ffa4ee4 100644 --- a/api/main.go +++ b/api/main.go @@ -3,10 +3,6 @@ package main // TODO: API functions that still need updating for Live databases // * diff - already updated to just return an error for live databases. needs testing though -// FIXME: Update the documented Upload() function return values on the API doc page. Currently it talks about -// returning the commit ID for the upload. We'll probably return that field with a blank value for live -// databases though. TBD. - // FIXME: After the API and webui pieces are done, figure out how the DB4S end // point and dio should be updated to use live databases too @@ -113,29 +109,29 @@ func main() { } // Our pages - http.Handle("/", gz.GzipHandler(handleWrapper(rootHandler))) - http.Handle("/changelog", gz.GzipHandler(handleWrapper(changeLogHandler))) - http.Handle("/changelog.html", gz.GzipHandler(handleWrapper(changeLogHandler))) - http.Handle("/v1/branches", gz.GzipHandler(handleWrapper(branchesHandler))) - http.Handle("/v1/columns", gz.GzipHandler(handleWrapper(columnsHandler))) - http.Handle("/v1/commits", gz.GzipHandler(handleWrapper(commitsHandler))) - http.Handle("/v1/databases", gz.GzipHandler(handleWrapper(databasesHandler))) - http.Handle("/v1/delete", gz.GzipHandler(handleWrapper(deleteHandler))) - http.Handle("/v1/diff", gz.GzipHandler(handleWrapper(diffHandler))) - http.Handle("/v1/download", gz.GzipHandler(handleWrapper(downloadHandler))) - http.Handle("/v1/execute", gz.GzipHandler(handleWrapper(executeHandler))) - http.Handle("/v1/indexes", gz.GzipHandler(handleWrapper(indexesHandler))) - http.Handle("/v1/metadata", gz.GzipHandler(handleWrapper(metadataHandler))) - http.Handle("/v1/query", gz.GzipHandler(handleWrapper(queryHandler))) - http.Handle("/v1/releases", gz.GzipHandler(handleWrapper(releasesHandler))) - http.Handle("/v1/tables", gz.GzipHandler(handleWrapper(tablesHandler))) - http.Handle("/v1/tags", gz.GzipHandler(handleWrapper(tagsHandler))) - http.Handle("/v1/upload", gz.GzipHandler(handleWrapper(uploadHandler))) - http.Handle("/v1/views", gz.GzipHandler(handleWrapper(viewsHandler))) - http.Handle("/v1/webpage", gz.GzipHandler(handleWrapper(webpageHandler))) + http.Handle("/", gz.GzipHandler(corsWrapper(rootHandler))) + http.Handle("/changelog", gz.GzipHandler(corsWrapper(changeLogHandler))) + http.Handle("/changelog.html", gz.GzipHandler(corsWrapper(changeLogHandler))) + http.Handle("/v1/branches", gz.GzipHandler(corsWrapper(branchesHandler))) + http.Handle("/v1/columns", gz.GzipHandler(corsWrapper(columnsHandler))) + http.Handle("/v1/commits", gz.GzipHandler(corsWrapper(commitsHandler))) + http.Handle("/v1/databases", gz.GzipHandler(corsWrapper(databasesHandler))) + http.Handle("/v1/delete", gz.GzipHandler(corsWrapper(deleteHandler))) + http.Handle("/v1/diff", gz.GzipHandler(corsWrapper(diffHandler))) + http.Handle("/v1/download", gz.GzipHandler(corsWrapper(downloadHandler))) + http.Handle("/v1/execute", gz.GzipHandler(corsWrapper(executeHandler))) + http.Handle("/v1/indexes", gz.GzipHandler(corsWrapper(indexesHandler))) + http.Handle("/v1/metadata", gz.GzipHandler(corsWrapper(metadataHandler))) + http.Handle("/v1/query", gz.GzipHandler(corsWrapper(queryHandler))) + http.Handle("/v1/releases", gz.GzipHandler(corsWrapper(releasesHandler))) + http.Handle("/v1/tables", gz.GzipHandler(corsWrapper(tablesHandler))) + http.Handle("/v1/tags", gz.GzipHandler(corsWrapper(tagsHandler))) + http.Handle("/v1/upload", gz.GzipHandler(corsWrapper(uploadHandler))) + http.Handle("/v1/views", gz.GzipHandler(corsWrapper(viewsHandler))) + http.Handle("/v1/webpage", gz.GzipHandler(corsWrapper(webpageHandler))) // favicon.ico - http.Handle("/favicon.ico", gz.GzipHandler(handleWrapper(func(w http.ResponseWriter, r *http.Request) { + http.Handle("/favicon.ico", gz.GzipHandler(corsWrapper(func(w http.ResponseWriter, r *http.Request) { logReq(r, "-") http.ServeFile(w, r, filepath.Join(com.Conf.Web.BaseDir, "webui", "favicon.ico")) }))) @@ -329,9 +325,8 @@ func extractUserFromClientCert(w http.ResponseWriter, r *http.Request) (userAcc return } -// handleWrapper does nothing useful except interface between types -// TODO: Get rid of this, as it shouldn't be needed -func handleWrapper(fn http.HandlerFunc) http.HandlerFunc { +// corsWrapper sets a general allow for all our api calls +func corsWrapper(fn http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { // Enable CORS (https://enable-cors.org) w.Header().Set("Access-Control-Allow-Origin", "*") @@ -361,6 +356,5 @@ func jsonErr(w http.ResponseWriter, msg string, statusCode int) { // logReq writes an entry for the incoming request to the request log func logReq(r *http.Request, loggedInUser string) { fmt.Fprintf(reqLog, "%v - %s [%s] \"%s %s %s\" \"-\" \"-\" \"%s\" \"%s\"\n", r.RemoteAddr, - loggedInUser, time.Now().Format(time.RFC3339Nano), r.Method, r.URL, r.Proto, - r.Referer(), r.Header.Get("User-Agent")) + loggedInUser, time.Now().Format(time.RFC3339Nano), r.Method, r.URL, r.Proto, r.Referer(), r.Header.Get("User-Agent")) } diff --git a/common/postgresql.go b/common/postgresql.go index b9f9ccb97..f45fa9335 100644 --- a/common/postgresql.go +++ b/common/postgresql.go @@ -96,6 +96,51 @@ func AddUser(auth0ID, userName, password, email, displayName, avatarURL string) return nil } +// ApiCallLog records an API call operation. Database name is optional, as not all API calls operate on a +// database. If a database name is provided however, then the database owner name *must* also be provided +func ApiCallLog(loggedInUser, dbOwner, dbName, operation, callerSw string) { + var dbQuery string + var err error + var commandTag pgx.CommandTag + if dbName != "" { + dbQuery = ` + WITH loggedIn AS ( + SELECT user_id + FROM users + WHERE lower(user_name) = lower($1) + ), + WITH owner AS ( + SELECT user_id + FROM users + WHERE lower(user_name) = lower($2) + ), d AS ( + SELECT db.db_id + FROM sqlite_databases AS db, owner + WHERE db.user_id = owner.user_id + AND db.db_name = $3) + INSERT INTO api_call_log (caller_id, db_owner_id, db_id, api_operation, api_caller_sw) + VALUES ((SELECT user_id FROM loggedIn), (SELECT user_id FROM owner), (SELECT db_id FROM d), $4, $5)` + commandTag, err = pdb.Exec(dbQuery, loggedInUser, dbOwner, dbName, operation, callerSw) + } else { + dbQuery = ` + WITH loggedIn AS ( + SELECT user_id + FROM users + WHERE lower(user_name) = lower($1) + ) + INSERT INTO api_call_log (caller_id, api_operation, api_caller_sw) + VALUES ((SELECT user_id FROM loggedIn), $2, $3)` + commandTag, err = pdb.Exec(dbQuery, loggedInUser, operation, callerSw) + } + if err != nil { + log.Printf("Adding api call log entry failed: %s", err) + return + } + if numRows := commandTag.RowsAffected(); numRows != 1 { + log.Printf("Wrong number of rows (%d) affected when adding api call entry for user '%s'", numRows, SanitiseLogString(loggedInUser)) + } +} + // APIKeySave saves a new API key to the PostgreSQL database func APIKeySave(key, loggedInUser string, dateCreated time.Time) error { // Make sure the API key isn't already in the database diff --git a/database/dbhub.sql b/database/dbhub.sql index 7629a7ad9..e140aba17 100644 --- a/database/dbhub.sql +++ b/database/dbhub.sql @@ -30,6 +30,28 @@ SET default_tablespace = ''; SET default_table_access_method = heap; +-- +-- Name: api_call_log; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.api_call_log ( + api_call_id bigint NOT NULL, + api_call_date timestamp with time zone DEFAULT now(), + caller_id bigint, + db_owner_id bigint, + db_id bigint, + api_operation text NOT NULL, + api_caller_sw text +); + + +-- +-- Name: COLUMN api_call_log.db_id; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.api_call_log.db_id IS 'This field must be nullable, as not all api calls act on a database'; + + -- -- Name: api_keys; Type: TABLE; Schema: public; Owner: - -- @@ -61,6 +83,25 @@ CREATE SEQUENCE public.api_keys_key_id_seq ALTER SEQUENCE public.api_keys_key_id_seq OWNED BY public.api_keys.key_id; +-- +-- Name: api_log_log_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.api_log_log_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: api_log_log_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.api_log_log_id_seq OWNED BY public.api_call_log.api_call_id; + + -- -- Name: database_downloads; Type: TABLE; Schema: public; Owner: - -- @@ -596,6 +637,13 @@ CREATE TABLE public.watchers ( ); +-- +-- Name: api_call_log api_call_id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.api_call_log ALTER COLUMN api_call_id SET DEFAULT nextval('public.api_log_log_id_seq'::regclass); + + -- -- Name: api_keys key_id; Type: DEFAULT; Schema: public; Owner: - -- @@ -1003,6 +1051,14 @@ ALTER TABLE ONLY public.api_keys ADD CONSTRAINT api_keys_users_user_id_fk FOREIGN KEY (user_id) REFERENCES public.users(user_id) ON UPDATE CASCADE ON DELETE SET NULL; +-- +-- Name: api_call_log api_log_users_user_id_fk; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.api_call_log + ADD CONSTRAINT api_log_users_user_id_fk FOREIGN KEY (caller_id) REFERENCES public.users(user_id); + + -- -- Name: database_downloads database_downloads_db_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --