diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 07fae56..88a4998 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,15 +16,15 @@ # limitations under the License. --- -default_stages: [commit, push] +default_stages: [pre-commit, pre-push] default_language_version: # force all unspecified python hooks to run python3 python: python3 -minimum_pre_commit_version: "1.20.0" +minimum_pre_commit_version: "3.4.0" repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.3.0 + rev: v5.0.0 hooks: - id: check-yaml - id: end-of-file-fixer diff --git a/README.md b/README.md index 39f6914..04b5ea8 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Apache Kibble is a tool to collect, aggregate and visualize data about any softw for the scanners to connect to, and provides the overall management of sources as well as the visualizations and API end points. - **Kibble scanners** ([kibble-scanners](https://github.com/apache/kibble-scanners)) - a collection of - scanning applications each designed to work with a specific type of resource (git repo, mailing list, + scanning applications each designed to work with a specific type of resource (git repo, mailing list, JIRA, etc) and push compiled data objects to the Kibble Server. ### Documentation @@ -30,6 +30,31 @@ service: [https://demo.kibble.apache.org/](https://demo.kibble.apache.org/). For installation steps see the [documentation](https://apache-kibble.readthedocs.io/en/latest/setup.html#installing-the-server). +### Packaging + +After installation of the build requirements + + pip install -q build + +build the project by running + + python -m build + +Find more information here: [Setuptools](https://setuptools.pypa.io/) and in project.toml file. + + +### Installation + +To install **Kibble-1** for development and/or testing from the checked-out code repository, run the following from the repository root: + + pip install -e '.[dev]' + +Find more information [Editable project mode](https://setuptools.pypa.io/en/latest/userguide/development_mode.html). + + +More TBD .. + + ### Contributing We welcome all contributions that improve the state of the Apache Kibble project. For contribution guidelines diff --git a/api/handler.py b/api/handler.py index ec0b72f..4161acf 100644 --- a/api/handler.py +++ b/api/handler.py @@ -33,6 +33,8 @@ import plugins.database import plugins.openapi +__version__ = "1.0.0" + # Compile valid API URLs from the pages library # Allow backwards compatibility by also accepting .lua URLs urls = [] @@ -55,7 +57,7 @@ class KibbleHTTPError(Exception): def __init__(self, code, message): self.code = code self.message = message - + class KibbleAPIWrapper: """ @@ -66,7 +68,7 @@ def __init__(self, path, func): self.API = KibbleOpenAPI self.path = path self.exception = KibbleHTTPError - + def __call__(self, environ, start_response, session): """Run the function, return response OR return stacktrace""" response = None @@ -89,7 +91,7 @@ def __call__(self, environ, start_response, session): "reason": "Invalid JSON: %s" % err }) return - + # Validate URL against OpenAPI specs try: self.API.validate(environ['REQUEST_METHOD'], self.path, formdata) @@ -101,7 +103,7 @@ def __call__(self, environ, start_response, session): "reason": err.message }) return - + # Call page with env, SR and form data try: response = self.func(self, environ, formdata, session) @@ -118,7 +120,7 @@ def __call__(self, environ, start_response, session): errHeader = errHeaders[err.code] if err.code in errHeaders else "400 Bad request" if __debug__: print("Set response header: %s." % ( errHeader ) ) #'Set-Cookie' - # traceBack(err) + # traceBack(err) start_response(errHeader, [ ('Content-Type', 'application/json')]) yield json.dumps({ @@ -126,7 +128,7 @@ def __call__(self, environ, start_response, session): "reason": err.message }, indent = 4) + "\n" return - + except Exception as err: traceback_output = traceBack(err) # We don't know if response has been given yet, try giving one, fail gracefully. @@ -139,16 +141,16 @@ def __call__(self, environ, start_response, session): "code": "500", "reason": '\n'.join(traceback_output) }) - + def traceBack(err): - print("Initial exception error: %s" % ( err ) ) + print("Initial exception error: %s" % ( err ) ) err_type, err_value, tb = sys.exc_info() traceback_output = ['API traceback:'] traceback_output += traceback.format_tb(tb) traceback_output.append('%s: %s' % (err_type.__name__, err_value)) - print("Error: traceback_output: %s" % (traceback_output)) + print("Error: traceback_output: %s" % (traceback_output)) return traceback_output - + def fourohfour(environ, start_response): """A very simple 404 handler""" start_response("404 Not Found", [ @@ -173,15 +175,15 @@ def application(environ, start_response): if m: callback = KibbleAPIWrapper(path, function) session = plugins.session.KibbleSession(DB, environ, config) - if __debug__: - print("Path %s setting in session %s header %s" % ( path, session, session.headers ) ) #'Set-Cookie' + #if __debug__: + # print("Path %s setting in session %s header %s" % ( path, session, session.headers ) ) #'Set-Cookie' a = 0 for bucket in callback(environ, start_response, session): if a == 0: #if __debug__: # print("Checking list type of bucket: %s %s" % ( type(bucket), bucket ) ) if isinstance(bucket, dict): - print("Added to session headers now %s" % ( session.headers ) ) + print("Added to session headers now %s" % ( session.headers ) ) session.headers.append(bucket) try: start_response("200 Okay", (session.headers) ) @@ -194,7 +196,7 @@ def application(environ, start_response): elif isinstance(bucket, bytes): yield bucket return - + for bucket in fourohfour(environ, start_response): yield bytes(bucket, encoding = 'utf-8') diff --git a/api/plugins/database.py b/api/plugins/database.py index b16fd75..c12b720 100644 --- a/api/plugins/database.py +++ b/api/plugins/database.py @@ -35,7 +35,7 @@ class _KibbleESWrapper(object): """ def __init__(self, ES): self.ES = ES - + def get(self, index, doc_type, id): return self.ES.get(index = index+'_'+doc_type, doc_type = '_doc', id = id) def exists(self, index, doc_type, id): @@ -73,7 +73,7 @@ class _KibbleESWrapperSeven(object): """ def __init__(self, ES): self.ES = ES - + def get(self, index, doc_type, id): return self.ES.get(index = index+'_'+doc_type, id = id) def exists(self, index, doc_type, id): @@ -101,13 +101,13 @@ def count(self, index, doc_type = '*', body = None): index = index+'_'+doc_type, body = body ) - + class _KibbleESWrapperEight(_KibbleESWrapperSeven): def __init__(self, ES): super().__init__(ES) # to replace key in body in queries self.replace = {'interval': 'calendar_interval'} # or fixed_interval - + def index(self, index, doc_type, id, body): if body is not None: body = self.ndict_replace(body, self.replace) @@ -116,10 +116,13 @@ def update(self, index, doc_type, id, body): if body is not None: body = self.ndict_replace(body, self.replace) return self.ES.update(index = index+'_'+doc_type, id = id, body = body) - + def search(self, index, doc_type, size = 100, scroll = None, _source_include = None, body = None): if body is not None: body = self.ndict_replace(body, self.replace) + if 'size' in body: + print("WARNING duplicate size: body size %s and size param: %s" % (body['size'], size) ) + #del body['size'] return self.ES.search( index = index+'_'+doc_type, size = size, @@ -134,7 +137,7 @@ def count(self, index, doc_type = '*', body = None): index = index+'_'+doc_type, body = body ) - + def ndict_replace(self, dict, replace): #print("original body/dict : %s." %(dict) ) ndict = NestedDict(dict) @@ -145,7 +148,7 @@ def ndict_replace(self, dict, replace): #print("replace %s matched in key %s " %(key, result) ) new_key = result new_nd[new_key] = value - new_dict = new_nd.to_dict(); + new_dict = new_nd.to_dict(); #print("replaced body/dict: %s." %(new_dict) ) return new_dict @@ -153,7 +156,7 @@ class KibbleDatabase(object): def __init__(self, config): self.config = config self.dbname = config['elasticsearch']['dbname'] - + defaultELConfig = { 'host': config['elasticsearch']['host'], 'port': int(config['elasticsearch']['port']), @@ -167,12 +170,12 @@ def __init__(self, config): defaultELConfig['verify_certs']: False defaultELConfig['url_prefix'] = config['elasticsearch']['uri'] if 'uri' in config['elasticsearch'] else '' defaultELConfig['http_auth'] = config['elasticsearch']['auth'] if 'auth' in config['elasticsearch'] else None - + self.ES = elasticsearch.Elasticsearch([ defaultELConfig ], max_retries=5, retry_on_timeout=True ) - + # IMPORTANT BIT: Figure out if this is ES < 6.x, 6.x or >= 7.x. # If so, we're using the new ES DB mappings, and need to adjust ALL # ES calls to match this. @@ -183,4 +186,3 @@ def __init__(self, config): self.ES = _KibbleESWrapperSeven(self.ES) elif self.ESVersion >= 6: self.ES = _KibbleESWrapper(self.ES) - diff --git a/api/plugins/session.py b/api/plugins/session.py index 681fb13..4ab5dfa 100644 --- a/api/plugins/session.py +++ b/api/plugins/session.py @@ -31,13 +31,13 @@ import time class KibbleSession(object): - + def getView(self, viewID): if self.DB.ES.exists(index=self.DB.dbname, doc_type="view", id = viewID): view = self.DB.ES.get(index=self.DB.dbname, doc_type="view", id = viewID) return view['_source']['sourceList'] return [] - + def subFilter(self, subfilter, view = []): if len(subfilter) == 0: return view @@ -56,7 +56,7 @@ def subFilter(self, subfilter, view = []): } }] } - + } } ) @@ -69,7 +69,7 @@ def subFilter(self, subfilter, view = []): if not sources: sources = ['x'] # blank return to not show eeeeverything return sources - + def subType(self, stype, view = []): if len(stype) == 0: return view @@ -95,7 +95,7 @@ def subType(self, stype, view = []): } ] } - + } } ) @@ -108,7 +108,7 @@ def subType(self, stype, view = []): if not sources: sources = ['x'] # blank return to not show eeeeverything return sources - + def logout(self): """Log out user and wipe cookie""" if self.user and self.cookie: @@ -128,12 +128,12 @@ def newCookie(self): cookies['kibble_session'] = cookie cookies['kibble_session']['expires'] = 86400 * 365 # Expire one year from now cookies['kibble_session']['HttpOnly'] = True; # no js write exposure - # cookies['kibble_session']['secure'] = True; # more secure + # cookies['kibble_session']['secure'] = True; # more secure self.headers.append(('Set-Cookie', cookies['kibble_session'].OutputString())) if __debug__: print("headers ", ( self.headers) ) return cookie - + def __init__(self, DB, environ, config): """ Loads the current user session or initiates a new session if @@ -144,11 +144,11 @@ def __init__(self, DB, environ, config): self.DB = DB self.headers = [('Content-Type', 'application/json; charset=utf-8')] self.cookie = None - + # Construct the URL we're visiting self.url = "%s://%s" % (environ['wsgi.url_scheme'], environ.get('HTTP_HOST', environ.get('SERVER_NAME'))) self.url += environ.get('SCRIPT_NAME', '/') - + # Get Kibble cookie cookie = None cookies = None @@ -190,6 +190,5 @@ def __init__(self, DB, environ, config): if not cookie: cookie = self.newCookie() self.cookie = cookie - if __debug__: - print("cookie found/set ", (cookie) ) - \ No newline at end of file + #if __debug__: + # print("cookie found/set ", (cookie) ) diff --git a/docs/source/setup.rst b/docs/source/setup.rst index c403599..652717f 100644 --- a/docs/source/setup.rst +++ b/docs/source/setup.rst @@ -113,7 +113,7 @@ following components installed and set up: - - certifi - - pyyaml - - bcrypt -- Gunicorn for Python 3.x (often called gunicorn3) or mod_wsgi +- Gunicorn for Python 3.x (often called gunicorn3), Waitress or mod_wsgi ########################################### Configuring and Priming the Kibble Instance @@ -167,12 +167,12 @@ be using the Apache HTTP Server and proxy to Gunicorn: gunicorn -w 10 -b 127.0.0.1:8000 handler:application -t 120 -D Alternatively use waitress, e.g. in development like this: - + :: cd /var/www/kibble/api/ waitress-serve --listen=*:8000 handler:application - + Once httpd is (re)started, you should be able to browse to your new Kibble instance. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..6c541e4 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,60 @@ +############################## +# Python packaging settings: # + +[build-system] +requires = ["setuptools", "setuptools-scm"] +build-backend = "setuptools.build_meta" + +[project] +name = "kibble-1" +description = "Apache Kibble is a tool to collect, aggregate and visualize data about any software project that uses commonly known tools." +# version is dynamic + +dependencies = [ + "python-dateutil", + "certifi", + "elasticsearch", + "PyYAML>=5.2", +] + +requires-python = ">=3.9, <4.0" + +authors = [ + { name = "Apache Software Foundation", email = "dev@kibble.apache.org" }, +] +maintainers = [ + { name = "Apache Software Foundation", email="dev@kibble.apache.org" }, +] +keywords = [ + "kibble", "data" ] + +license = { text = "Apache License, Version 2.0" } + +classifiers = [ + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Environment :: Console", + "Environment :: Web Environment", + "Framework :: Apache Kibble-1", + "License :: OSI Approved :: Apache Software License", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Topic :: System :: Monitoring" +] + +dynamic = [ + "readme", + "version", +] + +[project.urls] +repository = "https://github.com/apache/kibble-1.git" +"Bug Tracker" = "https://github.com/apache/kibble-1/issues" + +[tool.setuptools] +dynamic = { readme = { file = ["README.md"] }, version = { attr = "handler.__version__" } } +packages.find = { where = ["api"] } diff --git a/setup/requirements.txt b/setup/requirements.txt index 4cb13e0..c1cfe7c 100644 --- a/setup/requirements.txt +++ b/setup/requirements.txt @@ -2,6 +2,6 @@ certifi pyyaml bcrypt elasticsearch -pre-commit +pre_commit python-dateutil ndicts