Packaging environment using pyproject.toml; change configuration of pre-commit hooks; use __version__ for project build version.

.pre-commit-config.yaml

@@ -16,15 +16,15 @@
 # limitations under the License.
 
 ---
-default_stages: [commit, push]
+default_stages: [pre-commit, pre-push]
 default_language_version:
   # force all unspecified python hooks to run python3
   python: python3
-minimum_pre_commit_version: "1.20.0"
+minimum_pre_commit_version: "3.4.0"
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.3.0
+    rev: v5.0.0
     hooks:
       - id: check-yaml
       - id: end-of-file-fixer

README.md

@@ -12,7 +12,7 @@ Apache Kibble is a tool to collect, aggregate and visualize data about any softw
  for the scanners to connect to, and provides the overall management of sources as well as the
  visualizations and API end points.
 - **Kibble scanners** ([kibble-scanners](https://github.com/apache/kibble-scanners)) - a collection of
- scanning applications each designed to work with a specific type of resource (git repo, mailing list, 
+ scanning applications each designed to work with a specific type of resource (git repo, mailing list,
  JIRA, etc) and push compiled data objects to the Kibble Server.
 
 ### Documentation
@@ -30,6 +30,31 @@ service: [https://demo.kibble.apache.org/](https://demo.kibble.apache.org/).
 
 For installation steps see the [documentation](https://apache-kibble.readthedocs.io/en/latest/setup.html#installing-the-server).
 
+### Packaging
+
+After installation of the build requirements
+
+    pip install -q build
+
+build the project by running
+
+    python -m build
+
+Find more information here: [Setuptools](https://setuptools.pypa.io/) and in project.toml file.
+
+
+### Installation
+
+To install **Kibble-1** for development and/or testing from the checked-out code repository, run the following from the repository root:
+
+    pip install -e '.[dev]'
+
+Find more information [Editable project mode](https://setuptools.pypa.io/en/latest/userguide/development_mode.html).
+
+
+More TBD ..
+
+
 ### Contributing
 
 We welcome all contributions that improve the state of the Apache Kibble project. For contribution guidelines

api/handler.py

@@ -33,6 +33,8 @@
 import plugins.database
 import plugins.openapi
 
+__version__ = "1.0.0"
+
 # Compile valid API URLs from the pages library
 # Allow backwards compatibility by also accepting .lua URLs
 urls = []
@@ -55,7 +57,7 @@ class KibbleHTTPError(Exception):
     def __init__(self, code, message):
         self.code = code
         self.message = message
-        
+
 
 class KibbleAPIWrapper:
     """
@@ -66,7 +68,7 @@ def __init__(self, path, func):
         self.API = KibbleOpenAPI
         self.path = path
         self.exception = KibbleHTTPError
-     
+
     def __call__(self, environ, start_response, session):
         """Run the function, return response OR return stacktrace"""
         response = None
@@ -89,7 +91,7 @@ def __call__(self, environ, start_response, session):
                         "reason": "Invalid JSON: %s" % err
                     })
                     return
-                
+
             # Validate URL against OpenAPI specs
             try:
                 self.API.validate(environ['REQUEST_METHOD'], self.path, formdata)
@@ -101,7 +103,7 @@ def __call__(self, environ, start_response, session):
                     "reason": err.message
                 })
                 return
-            
+
             # Call page with env, SR and form data
             try:
                 response = self.func(self, environ, formdata, session)
@@ -118,15 +120,15 @@ def __call__(self, environ, start_response, session):
                 errHeader = errHeaders[err.code] if err.code in errHeaders else "400 Bad request"
                 if __debug__:
                     print("Set response header: %s."  % ( errHeader ) )  #'Set-Cookie'
-                    #    traceBack(err)  
+                    #    traceBack(err)
                 start_response(errHeader, [
                             ('Content-Type', 'application/json')])
                 yield json.dumps({
                     "code": err.code,
                     "reason": err.message
                 }, indent = 4) + "\n"
                 return
-            
+
         except Exception as err:
             traceback_output = traceBack(err)
             # We don't know if response has been given yet, try giving one, fail gracefully.
@@ -139,16 +141,16 @@ def __call__(self, environ, start_response, session):
                 "code": "500",
                 "reason": '\n'.join(traceback_output)
             })
-    
+
 def traceBack(err):
-    print("Initial exception error: %s"  % ( err ) ) 
+    print("Initial exception error: %s"  % ( err ) )
     err_type, err_value, tb = sys.exc_info()
     traceback_output = ['API traceback:']
     traceback_output += traceback.format_tb(tb)
     traceback_output.append('%s: %s' % (err_type.__name__, err_value))
-    print("Error: traceback_output: %s" % (traceback_output)) 
+    print("Error: traceback_output: %s" % (traceback_output))
     return traceback_output
-        
+
 def fourohfour(environ, start_response):
     """A very simple 404 handler"""
     start_response("404 Not Found", [
@@ -173,15 +175,15 @@ def application(environ, start_response):
         if m:
             callback = KibbleAPIWrapper(path, function)
             session = plugins.session.KibbleSession(DB, environ, config)
-            if __debug__:
-                print("Path %s setting in session %s header %s"  % ( path, session, session.headers ) )  #'Set-Cookie'
+            #if __debug__:
+            #    print("Path %s setting in session %s header %s"  % ( path, session, session.headers ) )  #'Set-Cookie'
             a = 0
             for bucket in callback(environ, start_response, session):
                 if a == 0:
                     #if __debug__:
                     #    print("Checking list type of bucket: %s %s"  % ( type(bucket), bucket )  )
                     if isinstance(bucket, dict):
-                        print("Added to session headers now %s"  % ( session.headers ) )  
+                        print("Added to session headers now %s"  % ( session.headers ) )
                         session.headers.append(bucket)
                     try:
                         start_response("200 Okay", (session.headers) )
@@ -194,7 +196,7 @@ def application(environ, start_response):
                 elif isinstance(bucket, bytes):
                     yield bucket
             return
-            
+
     for bucket in fourohfour(environ, start_response):
         yield bytes(bucket, encoding = 'utf-8')
 

api/plugins/database.py

@@ -35,7 +35,7 @@ class _KibbleESWrapper(object):
     """
     def __init__(self, ES):
         self.ES = ES
-    
+
     def get(self, index, doc_type, id):
         return self.ES.get(index = index+'_'+doc_type, doc_type = '_doc', id = id)
     def exists(self, index, doc_type, id):
@@ -73,7 +73,7 @@ class _KibbleESWrapperSeven(object):
     """
     def __init__(self, ES):
         self.ES = ES
-    
+
     def get(self, index, doc_type, id):
         return self.ES.get(index = index+'_'+doc_type, id = id)
     def exists(self, index, doc_type, id):
@@ -101,13 +101,13 @@ def count(self, index, doc_type = '*', body = None):
             index = index+'_'+doc_type,
             body = body
             )
-    
+
 class _KibbleESWrapperEight(_KibbleESWrapperSeven):
     def __init__(self, ES):
         super().__init__(ES)
         # to replace key in body in queries
         self.replace = {'interval': 'calendar_interval'} # or fixed_interval
-        
+
     def index(self, index, doc_type, id, body):
         if body is not None:
             body = self.ndict_replace(body, self.replace)
@@ -116,10 +116,13 @@ def update(self, index, doc_type, id, body):
         if body is not None:
             body = self.ndict_replace(body, self.replace)
         return self.ES.update(index = index+'_'+doc_type, id = id, body = body)
-        
+
     def search(self, index, doc_type, size = 100, scroll = None, _source_include = None, body = None):
         if body is not None:
             body = self.ndict_replace(body, self.replace)
+        if 'size' in body:
+            print("WARNING duplicate size: body size %s and size param: %s" % (body['size'], size) )
+            #del body['size']
         return self.ES.search(
             index = index+'_'+doc_type,
             size = size,
@@ -134,7 +137,7 @@ def count(self, index, doc_type = '*', body = None):
             index = index+'_'+doc_type,
             body = body
             )
-         
+
     def ndict_replace(self, dict, replace):
         #print("original body/dict : %s." %(dict) )
         ndict = NestedDict(dict)
@@ -145,15 +148,15 @@ def ndict_replace(self, dict, replace):
             #print("replace %s matched in key %s " %(key, result) )
             new_key = result
             new_nd[new_key] = value
-        new_dict =  new_nd.to_dict(); 
+        new_dict =  new_nd.to_dict();
         #print("replaced body/dict: %s." %(new_dict) )
         return new_dict
 
 class KibbleDatabase(object):
     def __init__(self, config):
         self.config = config
         self.dbname = config['elasticsearch']['dbname']
-        
+
         defaultELConfig = {
             'host': config['elasticsearch']['host'],
             'port': int(config['elasticsearch']['port']),
@@ -167,12 +170,12 @@ def __init__(self, config):
            defaultELConfig['verify_certs']: False
            defaultELConfig['url_prefix'] = config['elasticsearch']['uri'] if 'uri' in config['elasticsearch'] else ''
            defaultELConfig['http_auth'] = config['elasticsearch']['auth'] if 'auth' in config['elasticsearch'] else None
-            
+
         self.ES = elasticsearch.Elasticsearch([ defaultELConfig ],
                 max_retries=5,
                 retry_on_timeout=True
             )
-        
+
         # IMPORTANT BIT: Figure out if this is ES < 6.x, 6.x or >= 7.x.
         # If so, we're using the new ES DB mappings, and need to adjust ALL
         # ES calls to match this.
@@ -183,4 +186,3 @@ def __init__(self, config):
             self.ES = _KibbleESWrapperSeven(self.ES)
         elif self.ESVersion >= 6:
             self.ES = _KibbleESWrapper(self.ES)
-

api/plugins/session.py

@@ -31,13 +31,13 @@
 import time
 
 class KibbleSession(object):
-    
+
     def getView(self, viewID):
         if self.DB.ES.exists(index=self.DB.dbname, doc_type="view", id = viewID):
             view = self.DB.ES.get(index=self.DB.dbname, doc_type="view", id = viewID)
             return view['_source']['sourceList']
         return []
-    
+
     def subFilter(self, subfilter, view = []):
         if len(subfilter) == 0:
             return view
@@ -56,7 +56,7 @@ def subFilter(self, subfilter, view = []):
                                 }
                             }]
                         }
-                        
+
                     }
                 }
             )
@@ -69,7 +69,7 @@ def subFilter(self, subfilter, view = []):
         if not sources:
             sources = ['x'] # blank return to not show eeeeverything
         return sources
-    
+
     def subType(self, stype, view = []):
         if len(stype) == 0:
             return view
@@ -95,7 +95,7 @@ def subType(self, stype, view = []):
                                 }
                             ]
                         }
-                        
+
                     }
                 }
             )
@@ -108,7 +108,7 @@ def subType(self, stype, view = []):
         if not sources:
             sources = ['x'] # blank return to not show eeeeverything
         return sources
-            
+
     def logout(self):
         """Log out user and wipe cookie"""
         if self.user and self.cookie:
@@ -128,12 +128,12 @@ def newCookie(self):
         cookies['kibble_session'] = cookie
         cookies['kibble_session']['expires'] = 86400 * 365 # Expire one year from now
         cookies['kibble_session']['HttpOnly'] = True; # no js write exposure
-        # cookies['kibble_session']['secure'] = True; # more secure 
+        # cookies['kibble_session']['secure'] = True; # more secure
         self.headers.append(('Set-Cookie', cookies['kibble_session'].OutputString()))
         if __debug__:
             print("headers ",  ( self.headers) )
         return cookie
-        
+
     def __init__(self, DB, environ, config):
         """
         Loads the current user session or initiates a new session if
@@ -144,11 +144,11 @@ def __init__(self, DB, environ, config):
         self.DB = DB
         self.headers = [('Content-Type', 'application/json; charset=utf-8')]
         self.cookie = None
-          
+
         # Construct the URL we're visiting
         self.url = "%s://%s" % (environ['wsgi.url_scheme'], environ.get('HTTP_HOST', environ.get('SERVER_NAME')))
         self.url += environ.get('SCRIPT_NAME', '/')
-        
+
         # Get Kibble cookie
         cookie = None
         cookies = None
@@ -190,6 +190,5 @@ def __init__(self, DB, environ, config):
             if not cookie:
                 cookie = self.newCookie()
         self.cookie = cookie
-        if __debug__:
-            print("cookie found/set ",  (cookie) )
-
+        #if __debug__:
+        #    print("cookie found/set ",  (cookie) )

docs/source/setup.rst

@@ -113,7 +113,7 @@ following components installed and set up:
 - - certifi
 - - pyyaml
 - - bcrypt
-- Gunicorn for Python 3.x (often called gunicorn3) or mod_wsgi
+- Gunicorn for Python 3.x (often called gunicorn3), Waitress or mod_wsgi
 
 ###########################################
 Configuring and Priming the Kibble Instance
@@ -167,12 +167,12 @@ be using the Apache HTTP Server and proxy to Gunicorn:
    gunicorn -w 10 -b 127.0.0.1:8000 handler:application -t 120 -D
 
    Alternatively use waitress, e.g. in development like this:
-   
+
 ::
 
    cd /var/www/kibble/api/
    waitress-serve --listen=*:8000 handler:application
-   
+
    Once httpd is (re)started, you should be able to browse to your new
 Kibble instance.