diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md deleted file mode 100644 index 04b98a9f49..0000000000 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ /dev/null @@ -1,27 +0,0 @@ -## Description - -Summary of the changes introduced in this PR. Try to use bullet points as much as possible. - -## Look & Feel - -This section can contain example pictures for UI, Input/Output for CLI, Request / Response for API endpoint, etc. - -## How to test - -Steps to test it or name of the tests functions. - -The library [flexmeasures-client](https://github.com/FlexMeasures/flexmeasures-client/) can be useful to showcase new features. For example, -it can be used to set some example data to be used in a new UI feature. - -## Further Improvements - -Potential improvements to be done in the same PR or follow up Issues/Discussions/PRs. - -## Related Items - -Mention if this PR closes an Issue or Project. - ---- - -- [ ] I agree to contribute to the project under Apache 2 License. -- [ ] To the best of my knowledge, the proposed patch is not based on code under GPL or other license that is incompatible with FlexMeasures diff --git a/.github/issue-branch.yml b/.github/issue-branch.yml deleted file mode 100644 index c8cba5e60a..0000000000 --- a/.github/issue-branch.yml +++ /dev/null @@ -1,2 +0,0 @@ -openDraftPR: true -autoCloseIssue: true diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index 4d80c39f7e..0000000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,61 +0,0 @@ ---- -name: build-docker-image - -on: - pull_request: - types: - - opened - - synchronize - push: - branches: - - main - -jobs: - build: - name: Build Docker Image - runs-on: ubuntu-latest - services: - postgres: - env: - POSTGRES_DB: flexmeasures_test - POSTGRES_PASSWORD: flexmeasures_test - POSTGRES_USER: flexmeasures_test - image: postgres:latest - options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Build Docker Image - run: docker build -t flexmeasures:latest -f Dockerfile . - - name: Generate random secret key - run: echo "SECRET_KEY=$(python3 -c 'import secrets; print(secrets.token_hex(24))')" - >> .env - - name: Export SQLALCHEMY_DATABASE_URI - run: echo "SQLALCHEMY_DATABASE_URI=postgresql://flexmeasures_test:flexmeasures_test@127.0.0.1:5432/flexmeasures_test" - >> .env - - name: Keep running flexmeasures container in background - run: docker run -t -d --env-file .env --network=host --name fm-container flexmeasures:latest - - name: Execute database upgrade - run: docker exec --env-file .env fm-container flexmeasures - db upgrade - - name: Add toy user - run: docker exec --env-file .env fm-container flexmeasures - add toy-account - - name: Generate prices dummy data - run: .github/workflows/generate-dummy-price.sh - - name: Copy prices dummy data - run: docker cp prices-tomorrow.csv fm-container:/app/prices-tomorrow.csv - - name: Add beliefs - run: docker exec --env-file .env fm-container flexmeasures - add beliefs --sensor 1 --source toy-user prices-tomorrow.csv --timezone Europe/Amsterdam - - name: Export TOMORROW - run: echo "TOMORROW=$(date --date="next day" '+%Y-%m-%d')" - >> $GITHUB_ENV - - name: Add schedule - run: docker exec --env-file .env fm-container flexmeasures - add schedule for-storage --sensor 2 --consumption-price-sensor 1 - --start ${TOMORROW}T07:00+01:00 --duration PT12H - --soc-at-start 50% --roundtrip-efficiency 90% diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml deleted file mode 100644 index 96f44a743e..0000000000 --- a/.github/workflows/deploy.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: deploy-to-staging - -on: - push: - branches: - - main - -jobs: - deploy: - name: "Deploy (main to staging)" - runs-on: ubuntu-latest - steps: - - name: Wait for tests to pass - uses: lewagon/wait-on-check-action@v0.2 - with: - ref: ${{ github.ref }} - # check-name: "Test (on Python3.8)" # name of the job we wait for (omit to wait for all checks) - running-workflow-name: "Deploy (main to staging)" # name of the check that will wait for other checks - repo-token: ${{ secrets.GITHUB_TOKEN }} - wait-interval: 20 # seconds - - uses: actions/checkout@v3 - with: - fetch-depth: '0' - ref: 'main' - - name: Install SSH key - uses: shimataro/ssh-key-action@v2 - with: - key: ${{ secrets.SSH_DEPLOYMENT_KEY }} # private ssh key - known_hosts: ${{ secrets.KNOWN_DEPLOYMENT_HOSTS }} # make via ssh-keyscan -t rsa - - run: ci/DEPLOY.sh - env: - BRANCH_NAME: main - STAGING_REMOTE_REPO: ${{ secrets.STAGING_REMOTE_REPO }} \ No newline at end of file diff --git a/.github/workflows/generate-dummy-price.sh b/.github/workflows/generate-dummy-price.sh deleted file mode 100755 index 06ee670454..0000000000 --- a/.github/workflows/generate-dummy-price.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -set -e -set -x - -TOMORROW=$(date --date="next day" '+%Y-%m-%d') - -echo "Hour,Price -${TOMORROW}T00:00:00,10 -${TOMORROW}T01:00:00,11 -${TOMORROW}T02:00:00,12 -${TOMORROW}T03:00:00,15 -${TOMORROW}T04:00:00,18 -${TOMORROW}T05:00:00,17 -${TOMORROW}T06:00:00,10.5 -${TOMORROW}T07:00:00,9 -${TOMORROW}T08:00:00,9.5 -${TOMORROW}T09:00:00,9 -${TOMORROW}T10:00:00,8.5 -${TOMORROW}T11:00:00,10 -${TOMORROW}T12:00:00,8 -${TOMORROW}T13:00:00,5 -${TOMORROW}T14:00:00,4 -${TOMORROW}T15:00:00,4 -${TOMORROW}T16:00:00,5.5 -${TOMORROW}T17:00:00,8 -${TOMORROW}T18:00:00,12 -${TOMORROW}T19:00:00,13 -${TOMORROW}T20:00:00,14 -${TOMORROW}T21:00:00,12.5 -${TOMORROW}T22:00:00,10 -${TOMORROW}T23:00:00,7" > prices-tomorrow.csv diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml deleted file mode 100644 index feed2895eb..0000000000 --- a/.github/workflows/lint-and-test.yml +++ /dev/null @@ -1,83 +0,0 @@ -name: lint-and-test - -on: - push: - pull_request: - types: - - opened -jobs: - check: - runs-on: ubuntu-latest - name: Check (on Python 3.9) - steps: - - uses: actions/setup-python@v4 - with: - python-version: 3.9 - - uses: actions/checkout@v3 - - uses: pre-commit/action@v3.0.0 - - test: - needs: check - # fixed for now due to problems with 22.04 (see #551), try -latest sometime - runs-on: ubuntu-20.04 - strategy: - fail-fast: false - matrix: - py_version: [ "3.8", "3.9", "3.10", "3.11" ] - include: - - python-version: "3.9" - coverage: yes - name: "Test (on Python ${{ matrix.py_version }})" - steps: - - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.py_version }} - - name: Check out src from Git - uses: actions/checkout@v3 - - name: Get history and tags for SCM versioning to work - run: | - git fetch --prune --unshallow - git fetch --depth=1 origin +refs/tags/*:refs/tags/* - - name: "Caching for dependencies (.txt) - restore existing or ensure new cache will be made" - uses: actions/cache@v2 - id: cache - with: - path: ${{ env.pythonLocation }} - # manually disable a cache if needed by (re)setting CACHE_DATE - key: ${{ runner.os }}-pip-${{ env.pythonLocation }}-${{ SECRETS.CACHE_DATE }}-${{ hashFiles('**/requirements/**/*.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - - run: | - ci/setup-postgres.sh - sudo apt-get -y install coinor-cbc - - name: Install FlexMeasures & exact dependencies for tests - run: make install-for-test - if: github.event_name == 'push' && steps.cache.outputs.cache-hit != 'true' - - name: Install FlexMeasures & latest dependencies for tests - run: make install-for-test pinned=no - if: github.event_name == 'pull_request' - - name: Run all tests except those marked to be skipped by GitHub AND record coverage - run: pytest -v -m "not skip_github" --cov=flexmeasures --cov-branch --cov-report=lcov - - name: Coveralls - uses: coverallsapp/github-action@v2 - if: ${{ matrix.coverage == 'yes' }} - env: - PGHOST: 127.0.0.1 - PGPORT: 5432 - PGUSER: flexmeasures_test - PGDB: flexmeasures_test - PGPASSWORD: flexmeasures_test - - services: - # Label used to access the service container - postgres: - # Docker Hub image - image: postgres:12.5 - env: - POSTGRES_USER: flexmeasures_test - POSTGRES_PASSWORD: flexmeasures_test - POSTGRES_DB: flexmeasures_test - ports: - - 5432:5432 - # needed because the postgres container does not provide a healthcheck - options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 diff --git a/.github/workflows/sync-fork.yml b/.github/workflows/sync-fork.yml new file mode 100644 index 0000000000..42ba380546 --- /dev/null +++ b/.github/workflows/sync-fork.yml @@ -0,0 +1,89 @@ +name: Sync fork + +on: + workflow_dispatch: {} + schedule: + - cron: "15 3 * * *" # Run every day at 3:15 UTC + +jobs: + sync: + runs-on: ubuntu-latest + steps: + - name: Checkout fork's default branch + uses: actions/checkout@v4 + with: + fetch-depth: 0 + path: "fork" + token: ${{ secrets.GH_PAT }} + - name: Checkout fork's configuration branch + uses: actions/checkout@v4 + with: + path: "configuration" + ref: "renovate-and-workflow-files" + token: ${{ secrets.GH_PAT }} + - name: Determine Upstream clone URL + id: upstream-repo-clone-url + uses: actions/github-script@v7 + with: + script: | + const { data } = await github.rest.repos.get({ + owner: context.repo.owner, + repo: context.repo.repo, + }); + if (data.fork) { + return data.parent.clone_url; + } else { + throw new Error('This repository is not a fork.'); + } + result-encoding: string + - name: Determine Upstream default branch + id: upstream-repo-default-branch + uses: actions/github-script@v7 + with: + script: | + const { data } = await github.rest.repos.get({ + owner: context.repo.owner, + repo: context.repo.repo, + }); + if (data.fork) { + return data.parent.default_branch; + } else { + throw new Error('This repository is not a fork.'); + } + result-encoding: string + - name: Sync fork with upstream + run: | + set -ex + cd fork + git config --global user.name "github-actions[bot]" + git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" + git remote add upstream ${{ steps.upstream-repo-clone-url.outputs.result }} + git fetch upstream ${{ steps.upstream-repo-default-branch.outputs.result }} + UPSTREAM_MOST_RECENT_COMMIT_HASH=$(git log upstream/${{ steps.upstream-repo-default-branch.outputs.result }} -n 1 --format="%H") + PREVIOUS_SYNC_COMMIT_HASH=$(cat ../configuration/upstream_commit_hash) + if [ "$PREVIOUS_SYNC_COMMIT_HASH" = "$UPSTREAM_MOST_RECENT_COMMIT_HASH" ]; then + echo "No need to sync, already up-to-date" + exit 0 + fi + + git reset --hard upstream/${{ steps.upstream-repo-default-branch.outputs.result }} + # Enforce the usage of our own config (renovate.json5) + git rm renovate.json* || true + # Avoid problems where an existing .gitignore file would prevent committing our configuration files + git rm .gitignore || true + # Delete existing workflows, we don't need to run them in our fork + rm -rf .github || true + # Instead of using "cp -r", rsync allows us to exclude the .git directory + rsync -av --exclude '.git' ../configuration/ . + rm upstream_commit_hash + git add . + git commit -m "add Renovate and custom workflow files" + git push --force-with-lease + + cd ../configuration + # git config user.name "github-actions[bot]" + # git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + echo $UPSTREAM_MOST_RECENT_COMMIT_HASH > upstream_commit_hash + git add upstream_commit_hash + git commit -m "update commit hash to $UPSTREAM_MOST_RECENT_COMMIT_HASH" + git push diff --git a/.github/workflows/trivy-dependencies-submission.yml b/.github/workflows/trivy-dependencies-submission.yml new file mode 100644 index 0000000000..71f59af7e3 --- /dev/null +++ b/.github/workflows/trivy-dependencies-submission.yml @@ -0,0 +1,27 @@ +name: SBOM upload from Trivy + +on: + workflow_dispatch: {} + schedule: + - cron: "0 9 */5 * *" # Run every fifth day at 9 AM UTC + +jobs: + SBOM-upload: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Run Trivy vulnerability scanner in repo mode + uses: aquasecurity/trivy-action@0.23.0 + with: + scan-type: 'fs' + ignore-unfixed: true + format: 'github' + output: 'trivy-results.gsbom' + github-pat: ${{ secrets.GITHUB_TOKEN }} # this causes a curl call to upload the snapshot + + - name: Upload report file + uses: actions/upload-artifact@v4 + with: + name: trivy-results + path: trivy-results.gsbom diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 68ab1c2c98..0000000000 --- a/.gitignore +++ /dev/null @@ -1,53 +0,0 @@ -build -dist - -raw_data -instance -*venv/ -.eggs -.env -.vscode -!.vscode/settings.json -!.vscode/spellright.dict -.coverage.* -.mypy_cache/ -*.rdb -*.rdb.settings -*.pickle -.idea/ - -.env-flexmeasures/ -flexmeasures.egg-info -flexmeasures.log* -*.csv - -.cache -__pycache__ -.pytest_cache/ -.ipynb_checkpoints/ -notebooks/.ipynb_checkpoints/ - -flexmeasures/ui/static/documentation -documentation/img/screenshot_* -documentation/_autosummary/ -generic_asset_fm_user_ownership.sql - -uml_diagram.png -db_schema.png - -.coverage -htmlcov -test/* -profile_reports/* - -pyproject.toml -poetry.lock -.venv -.vscode/ - -.DS_Store - -.gitconfig.* - -/postgres-data -coverage.lcov \ No newline at end of file diff --git a/renovate.json5 b/renovate.json5 new file mode 100644 index 0000000000..53fe2ac327 --- /dev/null +++ b/renovate.json5 @@ -0,0 +1,12 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": [ + "config:recommended", + ":disableRateLimiting" + ], + "labels": ["dependencies", "depManager:{{{manager}}}"], + "vulnerabilityAlerts": { + "labels": ["security", "dependencies", "depManager:{{{manager}}}"], + }, + "forkProcessing": "enabled" +}