Add PS ingestion business logic

.github/workflows/main.yml

@@ -1,4 +1,4 @@
-name: Babiri CI
+name: Statsugiri CI
 
 on: [push]
 
@@ -9,7 +9,6 @@ jobs:
         strategy:
             matrix:
                 python-version: ["3.9"]
-                go-version: ["1.17.2"]
         steps:
             # checks-out the repository under $GITHUB_WORKSPACE
             - uses: actions/checkout@v2
@@ -18,16 +17,16 @@ jobs:
               with:
                   python-version: ${{ matrix.python-version }}
 
-            - name: Environment Setup
+            - name: Root Environment Setup
               run: |
-                  "${GITHUB_WORKSPACE}"/environment_setup/setup.sh
+                  pip install -r ./requirements.txt
 
             - name: Check formatting and linting
               run: |
                   ./formatting_scripts/check_formatting_ci.sh
 
     pytest:
-        name: Data Ingestion Python Tests
+        name: PS Ingestion Tests
         runs-on: ubuntu-latest
         strategy:
             matrix:
@@ -40,10 +39,12 @@ jobs:
               with:
                   python-version: ${{ matrix.python-version }}
 
-            - name: Environment Setup
+            - name: PS Ingestion Environment Setup
+              working-directory: ./src/ps_ingestion
               run: |
-                  "${GITHUB_WORKSPACE}"/environment_setup/setup.sh
+                  pip install -r requirements.txt
 
             - name: Run Pytest
+              working-directory: ./src/ps_ingestion
               run: |
-                  make drilbur-test
+                  pytest .

.gitignore

@@ -2,3 +2,6 @@
 .DS_Store
 .env
 __pycache__
+.pytest_cache
+.mypy_cache
+.vscode

.pylintrc

@@ -295,7 +295,7 @@ indent-after-paren=4
 indent-string='    '
 
 # Maximum number of characters on a single line.
-max-line-length=100
+max-line-length=120
 
 # Maximum number of lines in a module.
 max-module-lines=1000

Makefile

@@ -2,6 +2,6 @@
 format:
 	cd formatting_scripts && ./fix_formatting.sh
 
-# Run data pipeline tests
-drilbur-test:
-	cd src/data_ingestion && PYTHONDONTWRITEBYTECODE=1 python -m pytest -p no:cacheprovider . -vv
+# Run PS ingestion tests
+ps-ingestion-test:
+	cd src/ps_ingestion && PYTHONDONTWRITEBYTECODE=1 python -m pytest -p no:cacheprovider . -vv

environment_setup/setup.sh

@@ -36,4 +36,4 @@ check_pip
 install_python_dependencies
 printf "\n***Setup complete! Please set environment variables for ACCESS_KEY_ID, SECRET_ACCESS_KEY, and REGION_NAME for AWS authentication.***\n\n"
 
-exit 0
+exit 0

formatting_scripts/fix_formatting.sh

@@ -9,7 +9,6 @@
 
 CURR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 SRC_DIR="$CURR_DIR/../src"
-LINT_THRESHOLD=8
 
 # Function to check if formatting dependencies are available
 function check_dependencies () {
@@ -20,12 +19,6 @@ function check_dependencies () {
         printf "\n***Black formatter is not installed. Please run `setup.sh` first.***\n\n"
         exit 1
     fi
-    # Check if pylint available
-    pylint --version &>/dev/null
-    if [[ "$?" != 0 ]]; then
-        printf "\n***Pylint is not installed. Please run `setup.sh` first.***\n\n"
-        exit 1
-    fi
 }
 
 # Function to run black python formatting
@@ -40,18 +33,6 @@ function run_black_formatting () {
     fi
 }
 
-# Function to run lint check
-function run_pylint_linting () {
-    printf "Running Pylint to check Python linting...\n\n"
-    # Suppress messages
-    pylint --fail-under=$LINT_THRESHOLD $SRC_DIR/**/*.py --rcfile=$CURR_DIR/../.pylintrc $>/dev/null
-
-    if [[ "$?" != 0 ]]; then
-        printf "\n***Python linting did not pass threshold of 9.0! Run `../test_scripts/run_pylint.sh` for log***\n\n"
-        exit 1
-    fi
-}
-
 function run_eof_new_line(){
     printf "Adding missing new lines to end of files...\n\n"
 
@@ -67,7 +48,6 @@ function run_eof_new_line(){
 check_dependencies
 # Run formatting
 run_black_formatting
-run_pylint_linting
 run_eof_new_line
 
 exit 0

requirements.txt

@@ -0,0 +1,9 @@
+black==22.3.0
+click==8.1.3
+mypy==0.991
+mypy-extensions==0.4.3
+pathspec==0.10.3
+platformdirs==2.6.2
+tomli==2.0.1
+typed-ast==1.5.4
+typing_extensions==4.4.0