diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000..39bbd268 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,4 @@ +{ + "image": "mcr.microsoft.com/devcontainers/universal:2", + "features": {} +} diff --git a/.env.example b/.env.example new file mode 100644 index 00000000..752e03cc --- /dev/null +++ b/.env.example @@ -0,0 +1,2 @@ +# Replace with your actual OpenAI API key +OPENAI_API_KEY=your-api-key-here diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..1b889c33 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,133 @@ +# File: CODEOWNERS +* @Pythagora-io/maintainers + +# File: .github/ip_assignment.yml +assign_to_owner: true +license: FSL-1.1-MIT + +# File: .github/copyright_template.txt +Copyright (c) 2024 Pythagora Technologies Inc. All rights reserved. +This project is licensed under the terms of the FSL-1.1-MIT license. + +# File: LICENSE +FSL-1.1-MIT License + +Copyright (c) 2024 Pythagora Technologies Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +# Functional Source License, Version 1.1, MIT Future License + +## Abbreviation + +FSL-1.1-MIT + +## Notice + +Copyright 2024 Pythagora Technologies, Inc. + +## Terms and Conditions + +### Licensor ("We") + +The party offering the Software under these Terms and Conditions. + +### The Software + +The "Software" is each version of the software that we make available under +these Terms and Conditions, as indicated by our inclusion of these Terms and +Conditions with the Software. + +### License Grant + +Subject to your compliance with this License Grant and the Patents, +Redistribution and Trademark clauses below, we hereby grant you the right to +use, copy, modify, create derivative works, publicly perform, publicly display +and redistribute the Software for any Permitted Purpose identified below. + +### Permitted Purpose + +A Permitted Purpose is any purpose other than a Competing Use. A Competing Use +means making the Software available to others in a commercial product or +service that: + +1. substitutes for the Software; + +2. substitutes for any other product or service we offer using the Software + that exists as of the date we make the Software available; or + +3. offers the same or substantially similar functionality as the Software. + +Permitted Purposes specifically include using the Software: + +1. for your internal use and access; + +2. for non-commercial education; + +3. for non-commercial research; and + +4. in connection with professional services that you provide to a licensee + using the Software in accordance with these Terms and Conditions. + +### Patents + +To the extent your use for a Permitted Purpose would necessarily infringe our +patents, the license grant above includes a license under our patents. If you +make a claim against any party that the Software infringes or contributes to +the infringement of any patent, then your patent license to the Software ends +immediately. + +### Redistribution + +The Terms and Conditions apply to all copies, modifications and derivatives of +the Software. + +If you redistribute any copies, modifications or derivatives of the Software, +you must include a copy of or a link to these Terms and Conditions and not +remove any copyright notices provided in or with the Software. + +### Disclaimer + +THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR +PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. + +IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE +SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, +EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. + +### Trademarks + +Except for displaying the License Details and identifying us as the origin of +the Software, you have no right under these Terms and Conditions to use our +trademarks, trade names, service marks or product names. + +## Grant of Future License + +We hereby irrevocably grant you an additional license to use the Software under +the MIT license that is effective on the second anniversary of the date we make +the Software available. On or after that date, you may use the Software under +the MIT license, in which case the following will apply: + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/.github/CODE_OF_CONDUCT.md b/.github/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..a4486afa --- /dev/null +++ b/.github/CODE_OF_CONDUCT.md @@ -0,0 +1,133 @@ + +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity +and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness towards other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the + overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or + advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email + address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +[INSERT CONTACT METHOD]. +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series +of actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or +permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within +the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.0, available at +[https://www.contributor-covenant.org/version/2/0/code_of_conduct.html][v2.0]. + +Community Impact Guidelines were inspired by +[Mozilla's code of conduct enforcement ladder][Mozilla CoC]. + +For answers to common questions about this code of conduct, see the FAQ at +[https://www.contributor-covenant.org/faq][FAQ]. Translations are available +at [https://www.contributor-covenant.org/translations][translations]. + +[homepage]: https://www.contributor-covenant.org +[v2.0]: https://www.contributor-covenant.org/version/2/0/code_of_conduct.html +[Mozilla CoC]: https://github.com/mozilla/diversity +[FAQ]: https://www.contributor-covenant.org/faq +[translations]: https://www.contributor-covenant.org/translations diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md new file mode 100644 index 00000000..e69de29b diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml new file mode 100644 index 00000000..92041d17 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -0,0 +1,45 @@ +name: Bug Report +description: File a bug report +title: "[Bug]: " +labels: + - bug +body: + - type: markdown + attributes: + value: | + Thank you for taking the time to fill out this bug report! + + If you haven't already, please also check our [Frequently Asked Questions](https://github.com/Pythagora-io/gpt-pilot/wiki/Frequently-Asked-Questions) and [currently open issues](https://github.com/Pythagora-io/gpt-pilot/issues/) to see if the problem you have is already mentioned. If so, please comment on the the existing issue instead of creating a new one. + + We also have an [active community on Discord](https://discord.gg/HaqXugmxr9). If the issue is more about how to do something with GPT Pilot and not about a bug or problem, consider joining our Discord and discussing there, as it's possible other community members could help you! + - type: dropdown + id: client + attributes: + label: Version + description: Which version of GPT Pilot are you using? + options: + - VisualStudio Code extension + - Command-line (Python) version + validations: + required: true + - type: dropdown + id: os + attributes: + label: Operating System + description: Which operating system are you using? + options: + - Windows 10 + - Windows 11 + - MacOS + - Ubuntu Linux + - Linux (other) + validations: + required: true + - type: textarea + id: what-happened + attributes: + label: What happened? + description: Please describe the problem in as much detail as possible. If you have screenshots or screen recordings, please add them - they will help us figure out what's going on. + placeholder: "When I do [something], [a strange thing or a bug] happens instead of [what I expected]." + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/feature-request.yml b/.github/ISSUE_TEMPLATE/feature-request.yml new file mode 100644 index 00000000..460d2769 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature-request.yml @@ -0,0 +1,32 @@ +name: Feature request +description: Suggest a feature or improvement +title: "[Enhancement]: " +labels: + - enhancement +body: + - type: markdown + attributes: + value: | + Thank you for taking the time to suggest improvement to GPT Pilot! + + If you haven't already, please check our [Frequently Asked Questions](https://github.com/Pythagora-io/gpt-pilot/wiki/Frequently-Asked-Questions) and [currently open issues](https://github.com/Pythagora-io/gpt-pilot/issues/) to see if this is a commonly-asked feature. If so, please comment on the existing issue instead of creating a new one. + + We also have an [active community on Discord](https://discord.gg/HaqXugmxr9), so please consider joining, sharing your idea and starting a discussion there. + - type: dropdown + id: client + attributes: + label: Version + description: Which version of GPT Pilot does this apply? + options: + - VisualStudio Code extension + - Command-line (Python) version + validations: + required: true + - type: textarea + id: idea + attributes: + label: Suggestion + description: Please describe your suggestion for the improvement or new feature here. + placeholder: "It would be cool if GPT Pilot could do [something]." + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/howto.yml b/.github/ISSUE_TEMPLATE/howto.yml new file mode 100644 index 00000000..26d2f9a7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/howto.yml @@ -0,0 +1,47 @@ +name: How do I...? +description: Ask for help if you're stuck +title: "[Howto]: " +labels: + - question +body: + - type: markdown + attributes: + value: | + Thanks for trying out GPT Pilot! + + First, [check out our Frequently Asked Questions](https://github.com/Pythagora-io/gpt-pilot/wiki/Frequently-Asked-Questions) and our [video tutorials](https://www.youtube.com/watch?v=nlI7a20RPoE&list=PLbi3WiEeXr2wdppQ575zSLbcORou-Lxd1&index=2). There's a high chance your question has already been answered there. + + If not, it's best to ask the question [in our community Discord](https://discord.gg/HaqXugmxr9). We have an active and friendly community and you will likely get an answer quicker than if you post here. You can also check our [open issues](https://github.com/Pythagora-io/gpt-pilot/issues/) and see if what you're asking about was already discussed there. + + BTW: If you're wondering how to use GPT Pilot with a local LLM, please check our tutorial here: [Using GPT Pilot with Local LLMs](https://github.com/Pythagora-io/gpt-pilot/wiki/Using-GPT%E2%80%90Pilot-with-Local-LLMs). + - type: dropdown + id: client + attributes: + label: Version + description: Which version of GPT Pilot are you using? + options: + - VisualStudio Code extension + - Command-line (Python) version + validations: + required: true + - type: dropdown + id: os + attributes: + label: Operating System + description: Which operating system are you using? + options: + - Windows 10 + - Windows 11 + - MacOS + - Ubuntu Linux + - Linux (other) + validations: + required: true + - type: textarea + id: question + attributes: + label: Your question + description: Please describe your question in as much detail as possible. + placeholder: "How can I do [something] with GPT Pilot? I've checked the FAQ, video tutorials, open issues and Discord and there doesn't seem to be an answer to this." + validations: + required: true diff --git a/.github/copyright_template.txt b/.github/copyright_template.txt new file mode 100644 index 00000000..da1bc309 --- /dev/null +++ b/.github/copyright_template.txt @@ -0,0 +1,2 @@ +Copyright (c) 2024 Pythagora Technologies Inc. All rights reserved. +This project is licensed under the terms of the FSL-1.1-MIT license. diff --git a/.github/ip_assignment.yml b/.github/ip_assignment.yml new file mode 100644 index 00000000..d21016a5 --- /dev/null +++ b/.github/ip_assignment.yml @@ -0,0 +1,2 @@ +assign_to_owner: true +license: FSL-1.1-MIT diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..6dcaf7ce --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,35 @@ +name: Run unit tests + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main", "rewrite" ] + +jobs: + build: + + runs-on: ${{ matrix.os }} + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + python-version: ["3.9", "3.12"] + os: [ubuntu-latest, macos-latest, windows-latest] + + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install poetry + poetry install --with=dev + - name: Lint with ruff + run: poetry run ruff check --output-format github + - name: Check code style with ruff + run: poetry run ruff format --check --diff + - name: Test with pytest + run: poetry run pytest diff --git a/.gitignore b/.gitignore index 922a688c..de37dda5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,30 @@ +<<<<<<< HEAD +__pycache__/ +.venv/ +venv/ +.vscode/ +.idea/ +htmlcov/ +dist/ +workspace/ +pilot-env/ +venv/ + +.coverage +*.code-workspace +.*_cache +.env +*.pyc +*.db +*.db-shm +*.db-wal +config.json +poetry.lock +.DS_Store +*.log +pythagora-vs-code.vsix +======= node_modules/ uploads/ database_info.json +>>>>>>> 97f76c60971883c671587e44f13a9171b92f8ae7 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..975ececd --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,31 @@ +fail_fast: true +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.3.5 + hooks: + # Run the linter. + - id: ruff + args: [ --fix ] + # Run the formatter. + - id: ruff-format + - repo: local + hooks: + # Check there are no migrations missing + - id: alembic + name: alembic + stages: [commit] + types: [python] + entry: alembic -c core/db/alembic.ini check + language: system + pass_filenames: false + - repo: local + hooks: + # Run the tests + - id: pytest + name: pytest + stages: [commit] + types: [python] + entry: pytest + language: system + pass_filenames: false diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..e9125ce5 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,168 @@ +# Use Ubuntu 22.04 as the base image with multi-arch support +FROM ubuntu:22.04 + +# Set environment to prevent interactive prompts during builds +ENV DEBIAN_FRONTEND=noninteractive +ENV TZ=Etc/UTC + +# Use buildx args for multi-arch support +ARG TARGETPLATFORM +ARG BUILDPLATFORM + +# Update package list and install prerequisites +RUN apt-get update && apt-get install -y --no-install-recommends \ + software-properties-common \ + build-essential \ + curl \ + git \ + gnupg \ + tzdata \ + openssh-server \ + inotify-tools \ + vim \ + nano \ + && rm -rf /var/lib/apt/lists/* + +# Add deadsnakes PPA for Python 3.12 and install Python +RUN add-apt-repository ppa:deadsnakes/ppa -y && apt-get update && \ + apt-get install -y --no-install-recommends \ + python3.12 \ + python3.12-venv \ + python3.12-dev \ + python3-pip \ + && rm -rf /var/lib/apt/lists/* + + +# Set Python 3.12 as the default python3 and python +RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 1 && \ + update-alternatives --install /usr/bin/python python /usr/bin/python3 1 && \ + python --version + +RUN curl -fsSL https://deb.nodesource.com/setup_lts.x | bash - && \ + apt-get install -y nodejs && \ + node --version && npm --version + +# MongoDB installation with platform-specific approach +RUN case "$TARGETPLATFORM" in \ + "linux/amd64") \ + curl -fsSL https://www.mongodb.org/static/pgp/server-6.0.asc | gpg --dearmor -o /usr/share/keyrings/mongodb-archive-keyring.gpg && \ + echo "deb [arch=amd64 signed-by=/usr/share/keyrings/mongodb-archive-keyring.gpg] https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/6.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-6.0.list && \ + apt-get update && apt-get install -y mongodb-org \ + ;; \ + "linux/arm64"|"linux/arm64/v8") \ + curl -fsSL https://www.mongodb.org/static/pgp/server-6.0.asc | gpg --dearmor -o /usr/share/keyrings/mongodb-archive-keyring.gpg && \ + echo "deb [arch=arm64 signed-by=/usr/share/keyrings/mongodb-archive-keyring.gpg] https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/6.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-6.0.list && \ + apt-get update && apt-get install -y mongodb-org \ + ;; \ + *) \ + echo "Unsupported platform: $TARGETPLATFORM" && exit 1 \ + ;; \ + esac \ + && rm -rf /var/lib/apt/lists/* + +# Configure SSH +RUN mkdir -p /run/sshd \ + && sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin no/' /etc/ssh/sshd_config \ + && sed -i 's/#PubkeyAuthentication yes/PubkeyAuthentication yes/' /etc/ssh/sshd_config \ + && sed -i 's/#PasswordAuthentication yes/PasswordAuthentication no/' /etc/ssh/sshd_config \ + && sed -i 's/#ChallengeResponseAuthentication yes/ChallengeResponseAuthentication no/' /etc/ssh/sshd_config + +ENV PYTH_INSTALL_DIR=/pythagora + +# Set up work directory +WORKDIR ${PYTH_INSTALL_DIR}/pythagora-core + +# Add Python requirements +ADD requirements.txt . + +# Create and activate a virtual environment, then install dependencies +RUN python3 -m venv venv && \ + . venv/bin/activate && \ + pip install -r requirements.txt + +# Copy application files +ADD main.py . +ADD core core +ADD config-docker.json config.json + +# Set the virtual environment to be automatically activated +ENV VIRTUAL_ENV=${PYTH_INSTALL_DIR}/pythagora-core/venv +ENV PATH="$VIRTUAL_ENV/bin:$PATH" + +ENV PYTHAGORA_DATA_DIR=${PYTH_INSTALL_DIR}/pythagora-core/data/ +RUN mkdir -p data + +# Expose MongoDB and application ports +EXPOSE 27017 8000 + +# Create a group named "devusergroup" with a specific GID (1000, optional) +RUN groupadd -g 1000 devusergroup + +ARG USERNAME=devuser + +# Create a user named "devuser" with a specific UID (1000) and assign it to "devusergroup" +RUN useradd -m -u 1000 -g devusergroup -s /bin/bash $USERNAME + +# Add the user to sudoers for admin privileges +RUN echo "$USERNAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers + +# Create an embedded entrypoint script +ADD entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +RUN chown -R $USERNAME:devusergroup /pythagora + +# Copy SSH public key from secret +#RUN --mount=type=secret,id=ssh_public_key \ +# mkdir -p /home/${USERNAME}/.ssh \ +# && cat /run/secrets/ssh_public_key > /home/${USERNAME}/.ssh/authorized_keys \ +# && chown -R ${USERNAME}:devusergroup /home/${USERNAME}/.ssh \ +# && chmod 700 /home/${USERNAME}/.ssh \ +# && chmod 600 /home/${USERNAME}/.ssh/authorized_keys + +USER $USERNAME + +RUN npx @puppeteer/browsers install chrome@stable + +# add this before vscode... better caching of layers +ADD pythagora-vs-code.vsix /var/init_data/pythagora-vs-code.vsix + +RUN mkdir -p ~/.ssh && touch ~/.ssh/authorized_keys && chmod 600 ~/.ssh/authorized_keys + +# ARG commitHash +# # VS Code server installation with platform-specific handling +# RUN case "$TARGETPLATFORM" in \ +# "linux/amd64") \ +# mkdir -p ~/.vscode-server/cli/servers/Stable-${commitHash} && \ +# curl -fsSL https://update.code.visualstudio.com/commit:${commitHash}/server-linux-x64/stable -o server-linux-x64.tar.gz && \ +# tar -xz -f server-linux-x64.tar.gz -C ~/.vscode-server/cli/servers/Stable-${commitHash} && \ +# mv ~/.vscode-server/cli/servers/Stable-${commitHash}/vscode-server-linux-x64 ~/.vscode-server/cli/servers/Stable-${commitHash}/server \ +# ;; \ +# "linux/arm64"|"linux/arm64/v8") \ +# mkdir -p ~/.vscode-server/cli/servers/Stable-${commitHash} && \ +# curl -fsSL https://update.code.visualstudio.com/commit:${commitHash}/server-linux-arm64/stable -o server-linux-arm64.tar.gz && \ +# tar -xz -f server-linux-arm64.tar.gz -C ~/.vscode-server/cli/servers/Stable-${commitHash} && \ +# mv ~/.vscode-server/cli/servers/Stable-${commitHash}/vscode-server-linux-arm64 ~/.vscode-server/cli/servers/Stable-${commitHash}/server \ +# ;; \ +# *) \ +# echo "Unsupported platform: $TARGETPLATFORM" && exit 1 \ +# ;; \ +# esac + + +# Install VS Code extension (platform-agnostic) +# RUN ~/.vscode-server/cli/servers/Stable-${commitHash}/server/bin/code-server --install-extension pythagora-vs-code.vsix +ADD on-event-extension-install.sh /var/init_data/on-event-extension-install.sh + +# Create a workspace directory +RUN mkdir -p ${PYTH_INSTALL_DIR}/pythagora-core/workspace + +RUN mkdir -p /home/$USERNAME/.vscode-server/cli/servers + +USER root + +RUN chmod +x /var/init_data/on-event-extension-install.sh +RUN chown -R devuser: /var/init_data/ + +# Set the entrypoint to the main application +ENTRYPOINT ["/entrypoint.sh"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..74887def --- /dev/null +++ b/LICENSE @@ -0,0 +1,110 @@ +# Functional Source License, Version 1.1, MIT Future License + +## Abbreviation + +FSL-1.1-MIT + +## Notice + +Copyright 2024 Pythagora Technologies, Inc. + +## Terms and Conditions + +### Licensor ("We") + +The party offering the Software under these Terms and Conditions. + +### The Software + +The "Software" is each version of the software that we make available under +these Terms and Conditions, as indicated by our inclusion of these Terms and +Conditions with the Software. + +### License Grant + +Subject to your compliance with this License Grant and the Patents, +Redistribution and Trademark clauses below, we hereby grant you the right to +use, copy, modify, create derivative works, publicly perform, publicly display +and redistribute the Software for any Permitted Purpose identified below. + +### Permitted Purpose + +A Permitted Purpose is any purpose other than a Competing Use. A Competing Use +means making the Software available to others in a commercial product or +service that: + +1. substitutes for the Software; + +2. substitutes for any other product or service we offer using the Software + that exists as of the date we make the Software available; or + +3. offers the same or substantially similar functionality as the Software. + +Permitted Purposes specifically include using the Software: + +1. for your internal use and access; + +2. for non-commercial education; + +3. for non-commercial research; and + +4. in connection with professional services that you provide to a licensee + using the Software in accordance with these Terms and Conditions. + +### Patents + +To the extent your use for a Permitted Purpose would necessarily infringe our +patents, the license grant above includes a license under our patents. If you +make a claim against any party that the Software infringes or contributes to +the infringement of any patent, then your patent license to the Software ends +immediately. + +### Redistribution + +The Terms and Conditions apply to all copies, modifications and derivatives of +the Software. + +If you redistribute any copies, modifications or derivatives of the Software, +you must include a copy of or a link to these Terms and Conditions and not +remove any copyright notices provided in or with the Software. + +### Disclaimer + +THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR +PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. + +IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE +SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, +EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. + +### Trademarks + +Except for displaying the License Details and identifying us as the origin of +the Software, you have no right under these Terms and Conditions to use our +trademarks, trade names, service marks or product names. + +## Grant of Future License + +We hereby irrevocably grant you an additional license to use the Software under +the MIT license that is effective on the second anniversary of the date we make +the Software available. On or after that date, you may use the Software under +the MIT license, in which case the following will apply: + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index ac746db4..70813990 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,225 @@ +<<<<<<< HEAD +
+ +# πŸ§‘β€βœˆοΈ GPT PILOT πŸ§‘β€βœˆοΈ + +
+ +--- + +
+ +[![Discord Follow](https://dcbadge.vercel.app/api/server/HaqXugmxr9?style=flat)](https://discord.gg/HaqXugmxr9) +[![GitHub Repo stars](https://img.shields.io/github/stars/Pythagora-io/gpt-pilot?style=social)](https://github.com/Pythagora-io/gpt-pilot) +[![Twitter Follow](https://img.shields.io/twitter/follow/HiPythagora?style=social)](https://twitter.com/HiPythagora) + +
+ +--- + +
+Pythagora-io%2Fgpt-pilot | Trendshift +
+
+
+Pythagora-io%2Fgpt-pilot | Trendshift +
+ +
+
+ +
+ +### GPT Pilot doesn't just generate code, it builds apps! + +
+ +--- +
+ +[![See it in action](https://i3.ytimg.com/vi/4g-1cPGK0GA/maxresdefault.jpg)](https://youtu.be/4g-1cPGK0GA) + +(click to open the video in YouTube) (1:40min) + +
+ +--- + +
+ +Pythagora-io%2Fgpt-pilot | Trendshift + +
+ +GPT Pilot is the core technology for the [Pythagora VS Code extension](https://marketplace.visualstudio.com/items?itemName=PythagoraTechnologies.pythagora-vs-code) that aims to provide **the first real AI developer companion**. Not just an autocomplete or a helper for PR messages but rather a real AI developer that can write full features, debug them, talk to you about issues, ask for review, etc. + +--- + +πŸ“« If you would like to get updates on future releases or just get in touch, join our [Discord server](https://discord.gg/HaqXugmxr9) or you [can add your email here](http://eepurl.com/iD6Mpo). πŸ“¬ + +--- + + +* [πŸ”Œ Requirements](#-requirements) +* [🚦How to start using gpt-pilot?](#how-to-start-using-gpt-pilot) +* [πŸ”Ž Examples](#-examples) +* [🐳 How to start gpt-pilot in docker?](#-how-to-start-gpt-pilot-in-docker) +* [πŸ§‘β€πŸ’»οΈ CLI arguments](#-cli-arguments) +* [πŸ— How GPT Pilot works?](#-how-gpt-pilot-works) +* [πŸ•΄How's GPT Pilot different from _Smol developer_ and _GPT engineer_?](#hows-gpt-pilot-different-from-smol-developer-and-gpt-engineer) +* [🍻 Contributing](#-contributing) +* [πŸ”— Connect with us](#-connect-with-us) +* [🌟 Star history](#-star-history) + + +--- + +GPT Pilot aims to research how much LLMs can be utilized to generate fully working, production-ready apps while the developer oversees the implementation. + +**The main idea is that AI can write most of the code for an app (maybe 95%), but for the rest, 5%, a developer is and will be needed until we get full AGI**. + +If you are interested in our learnings during this project, you can check [our latest blog posts](https://blog.pythagora.ai/2024/02/19/gpt-pilot-what-did-we-learn-in-6-months-of-working-on-a-codegen-pair-programmer/). + +--- + +
+ +
+ +### **[πŸ‘‰ Examples of apps written by GPT Pilot πŸ‘ˆ](https://github.com/Pythagora-io/gpt-pilot/wiki/Apps-created-with-GPT-Pilot)** + +
+
+ +--- + +# πŸ”Œ Requirements + +- **Python 3.9+** + +# 🚦How to start using gpt-pilot? +πŸ‘‰ If you are using VS Code as your IDE, the easiest way to start is by downloading [GPT Pilot VS Code extension](https://marketplace.visualstudio.com/items?itemName=PythagoraTechnologies.pythagora-vs-code). πŸ‘ˆ + +Otherwise, you can use the CLI tool. + +### If you're new to GPT Pilot: + +After you have Python and (optionally) PostgreSQL installed, follow these steps: + +1. `git clone https://github.com/Pythagora-io/gpt-pilot.git` (clone the repo) +2. `cd gpt-pilot` (go to the repo folder) +3. `python3 -m venv venv` (create a virtual environment) +4. `source venv/bin/activate` (or on Windows `venv\Scripts\activate`) (activate the virtual environment) +5. `pip install -r requirements.txt` (install the dependencies) +6. `cp example-config.json config.json` (create `config.json` file) +7. Set your key and other settings in `config.json` file: + - LLM Provider (`openai`, `anthropic` or `groq`) key and endpoints (leave `null` for default) (note that Azure and OpenRouter are suppored via the `openai` setting) + - Your API key (if `null`, will be read from the environment variables) + - database settings: sqlite is used by default, PostgreSQL should also work + - optionally update `fs.ignore_paths` and add files or folders which shouldn't be tracked by GPT Pilot in workspace, useful to ignore folders created by compilers +8. `python main.py` (start GPT Pilot) + +All generated code will be stored in the folder `workspace` inside the folder named after the app name you enter upon starting the pilot. + +# πŸ”Ž [Examples](https://github.com/Pythagora-io/gpt-pilot/wiki/Pythagora-App-Lab) + +[Click here](https://github.com/Pythagora-io/gpt-pilot/wiki/Pythagora-App-Lab) to see examples of apps created with Pythagora. + +### PostgreSQL support + +GPT Pilot uses built-in SQLite database by default. If you want to use the PostgreSQL database, you need to additional install `asyncpg` and `psycopg2` packages: + +```bash +pip install asyncpg psycopg2 +``` + +Then, you need to update the `config.json` file to set `db.url` to `postgresql+asyncpg://:@/`. + +# πŸ§‘β€πŸ’»οΈ CLI arguments + +### List created projects (apps) + +```bash +python main.py --list +``` + +Note: for each project (app), this also lists "branches". Currently we only support having one branch (called "main"), and in the future we plan to add support for multiple project branches. + +### Load and continue from the latest step in a project (app) + +```bash +python main.py --project +``` + +### Load and continue from a specific step in a project (app) + +```bash +python main.py --project --step +``` + +Warning: this will delete all progress after the specified step! + +### Delete project (app) + +```bash +python main.py --delete +``` + +Delete project with the specified `app_id`. Warning: this cannot be undone! + +### Other command-line options + +There are several other command-line options that mostly support calling GPT Pilot from our VSCode extension. To see all the available options, use the `--help` flag: + +```bash +python main.py --help +``` + +# πŸ— How GPT Pilot works? +Here are the steps GPT Pilot takes to create an app: + +1. You enter the app name and the description. +2. **Product Owner agent** like in real life, does nothing. :) +3. **Specification Writer agent** asks a couple of questions to understand the requirements better if project description is not good enough. +4. **Architect agent** writes up technologies that will be used for the app and checks if all technologies are installed on the machine and installs them if not. +5. **Tech Lead agent** writes up development tasks that the Developer must implement. +6. **Developer agent** takes each task and writes up what needs to be done to implement it. The description is in human-readable form. +7. **Code Monkey agent** takes the Developer's description and the existing file and implements the changes. +8. **Reviewer agent** reviews every step of the task and if something is done wrong Reviewer sends it back to Code Monkey. +9. **Troubleshooter agent** helps you to give good feedback to GPT Pilot when something is wrong. +10. **Debugger agent** hate to see him, but he is your best friend when things go south. +11. **Technical Writer agent** writes documentation for the project. + +
+ +# πŸ•΄How's GPT Pilot different from _Smol developer_ and _GPT engineer_? + +- **GPT Pilot works with the developer to create a fully working production-ready app** - I don't think AI can (at least in the near future) create apps without a developer being involved. So, **GPT Pilot codes the app step by step** just like a developer would in real life. This way, it can debug issues as they arise throughout the development process. If it gets stuck, you, the developer in charge, can review the code and fix the issue. Other similar tools give you the entire codebase at once - this way, bugs are much harder to fix for AI and for you as a developer. +

+- **Works at scale** - GPT Pilot isn't meant to create simple apps but rather so it can work at any scale. It has mechanisms that filter out the code, so in each LLM conversation, it doesn't need to store the entire codebase in context, but it shows the LLM only the relevant code for the current task it's working on. Once an app is finished, you can continue working on it by writing instructions on what feature you want to add. + +# 🍻 Contributing +If you are interested in contributing to GPT Pilot, join [our Discord server](https://discord.gg/HaqXugmxr9), check out open [GitHub issues](https://github.com/Pythagora-io/gpt-pilot/issues), and see if anything interests you. We would be happy to get help in resolving any of those. The best place to start is by reviewing blog posts mentioned above to understand how the architecture works before diving into the codebase. + +## πŸ–₯ Development +Other than the research, GPT Pilot needs to be debugged to work in different scenarios. For example, we realized that the quality of the code generated is very sensitive to the size of the development task. When the task is too broad, the code has too many bugs that are hard to fix, but when the development task is too narrow, GPT also seems to struggle in getting the task implemented into the existing code. + +## πŸ“Š Telemetry +To improve GPT Pilot, we are tracking some events from which you can opt out at any time. You can read more about it [here](./docs/TELEMETRY.md). + +# πŸ”— Connect with us + +🌟 **If you find GPT Pilot useful, please consider [starring the repo](https://github.com/Pythagora-io/gpt-pilot)!** It helps us grow and continue improving the project. 🌟 + +πŸ’¬ **Need help or have questions?** +- Join our [Discord community](https://discord.gg/HaqXugmxr9) to connect with other users and our team. +- Visit our [Contact Us](https://github.com/Pythagora-io/gpt-pilot/wiki/Contact-Us) page for additional support. + +πŸ“– **Learn more about Pythagora & GPT Pilot:** +- Explore our [Wiki](https://github.com/Pythagora-io/gpt-pilot/wiki) for in-depth documentation. +- Check out our [FAQ](https://github.com/Pythagora-io/gpt-pilot/wiki/Frequently-Asked-Questions) for common questions and troubleshooting tips. +- Visit our [YouTube](https://www.youtube.com/@pythagoraa) channel for demos and how-to videos. +======= # Pythagora Lab Pythagora Lab is a web application designed for interactive viewing and analysis of data from SQLite databases. It allows users to upload SQLite databases, select projects, and explore detailed project states including tasks, files, LLM requests/responses, and more. The application is built using Node.js, Express, SQLite, EJS, and Bootstrap. @@ -89,4 +311,5 @@ Ensure you have the following installed on your computer: ``` Copyright (c) 2024. -``` \ No newline at end of file +``` +>>>>>>> 97f76c60971883c671587e44f13a9171b92f8ae7 diff --git a/config-docker.json b/config-docker.json new file mode 100644 index 00000000..7bee5dfc --- /dev/null +++ b/config-docker.json @@ -0,0 +1,136 @@ +{ + "llm": { + "openai": { + "base_url": null, + "api_key": null, + "connect_timeout": 60.0, + "read_timeout": 60.0, + "extra": null + }, + "anthropic": { + "base_url": null, + "api_key": null, + "connect_timeout": 60.0, + "read_timeout": 60.0, + "extra": null + } + }, + "agent": { + "default": { + "provider": "openai", + "model": "gpt-4o-2024-05-13", + "temperature": 0.5 + }, + "BugHunter.check_logs": { + "provider": "anthropic", + "model": "claude-3-5-sonnet-20241022", + "temperature": 0.5 + }, + "CodeMonkey": { + "provider": "anthropic", + "model": "claude-3-5-sonnet-20241022", + "temperature": 0.0 + }, + "CodeMonkey.code_review": { + "provider": "anthropic", + "model": "claude-3-5-sonnet-20240620", + "temperature": 0.0 + }, + "CodeMonkey.describe_files": { + "provider": "openai", + "model": "gpt-4o-mini-2024-07-18", + "temperature": 0.0 + }, + "Frontend": { + "provider": "anthropic", + "model": "claude-3-5-sonnet-20241022", + "temperature": 0.0 + }, + "get_relevant_files": { + "provider": "openai", + "model": "gpt-4o-2024-05-13", + "temperature": 0.5 + }, + "Developer.parse_task": { + "provider": "anthropic", + "model": "claude-3-5-sonnet-20241022", + "temperature": 0.0 + }, + "SpecWriter": { + "provider": "openai", + "model": "gpt-4-0125-preview", + "temperature": 0.0 + }, + "Developer.breakdown_current_task": { + "provider": "anthropic", + "model": "claude-3-5-sonnet-20241022", + "temperature": 0.5 + }, + "TechLead.plan_epic": { + "provider": "anthropic", + "model": "claude-3-5-sonnet-20240620", + "temperature": 0.5 + }, + "TechLead.epic_breakdown": { + "provider": "anthropic", + "model": "claude-3-5-sonnet-20241022", + "temperature": 0.5 + }, + "Troubleshooter.generate_bug_report": { + "provider": "anthropic", + "model": "claude-3-5-sonnet-20240620", + "temperature": 0.5 + }, + "Troubleshooter.get_run_command": { + "provider": "anthropic", + "model": "claude-3-5-sonnet-20240620", + "temperature": 0.0 + } + }, + "prompt": { + "paths": [ + "/pythagora/pythagora-core/core/prompts" + ] + }, + "log": { + "level": "DEBUG", + "format": "%(asctime)s %(levelname)s [%(name)s] %(message)s", + "output": "pythagora.log" + }, + "db": { + "url": "sqlite+aiosqlite:///data/database/pythagora.db", + "debug_sql": false + }, + "ui": { + "type": "plain" + }, + "fs": { + "type": "local", + "workspace_root": "/pythagora/pythagora-core/workspace", + "ignore_paths": [ + ".git", + ".gpt-pilot", + ".idea", + ".vscode", + ".next", + ".DS_Store", + "__pycache__", + "site-packages", + "node_modules", + "package-lock.json", + "venv", + ".venv", + "dist", + "build", + "target", + "*.min.js", + "*.min.css", + "*.svg", + "*.csv", + "*.log", + "go.sum", + "migration_lock.toml" + ], + "ignore_size_threshold": 50000 + } +} diff --git a/core/agents/__init__.py b/core/agents/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/core/agents/architect.py b/core/agents/architect.py new file mode 100644 index 00000000..758b7fe8 --- /dev/null +++ b/core/agents/architect.py @@ -0,0 +1,284 @@ +import json +from enum import Enum +from typing import Any, Optional + +from pydantic import BaseModel, Field + +from core.agents.base import BaseAgent +from core.agents.convo import AgentConvo +from core.agents.response import AgentResponse +from core.db.models import Specification +from core.llm.parser import JSONParser +from core.log import get_logger +from core.telemetry import telemetry +from core.templates.base import BaseProjectTemplate, NoOptions +from core.templates.example_project import EXAMPLE_PROJECTS +from core.templates.registry import ( + PROJECT_TEMPLATES, + ProjectTemplateEnum, +) + +ARCHITECTURE_STEP_NAME = "Project architecture" +WARN_SYSTEM_DEPS = ["docker", "kubernetes", "microservices"] +WARN_FRAMEWORKS = ["next.js", "vue", "vue.js", "svelte", "angular"] +WARN_FRAMEWORKS_URL = "https://github.com/Pythagora-io/gpt-pilot/wiki/Using-GPT-Pilot-with-frontend-frameworks" + +log = get_logger(__name__) + + +class AppType(str, Enum): + WEB = "web-app" + API = "api-service" + MOBILE = "mobile-app" + DESKTOP = "desktop-app" + CLI = "cli-tool" + + +# FIXME: all the reponse pydantic models should be strict (see config._StrictModel), also check if we +# can disallow adding custom Python attributes to the model +class SystemDependency(BaseModel): + name: str = Field( + None, + description="Name of the system dependency, for example Node.js or Python.", + ) + description: str = Field( + None, + description="One-line description of the dependency.", + ) + test: str = Field( + None, + description="Command line to test whether the dependency is available on the system.", + ) + required_locally: bool = Field( + None, + description="Whether this dependency must be installed locally (as opposed to connecting to cloud or other server)", + ) + + +class PackageDependency(BaseModel): + name: str = Field( + None, + description="Name of the package dependency, for example Express or React.", + ) + description: str = Field( + None, + description="One-line description of the dependency.", + ) + + +class Architecture(BaseModel): + app_type: AppType = Field( + AppType.WEB, + description="Type of the app to build.", + ) + system_dependencies: list[SystemDependency] = Field( + None, + description="List of system dependencies required to build and run the app.", + ) + package_dependencies: list[PackageDependency] = Field( + None, + description="List of framework/language-specific packages used by the app.", + ) + + +class TemplateSelection(BaseModel): + architecture: str = Field( + None, + description="General description of the app architecture.", + ) + template: Optional[ProjectTemplateEnum] = Field( + None, + description="Project template to use for the app, or null if no template is a good fit.", + ) + + +class Architect(BaseAgent): + agent_type = "architect" + display_name = "Architect" + + async def run(self) -> AgentResponse: + spec = self.current_state.specification.clone() + + if spec.example_project: + self.prepare_example_project(spec) + else: + await self.plan_architecture(spec) + + await self.check_system_dependencies(spec) + + self.next_state.specification = spec + telemetry.set("templates", spec.templates) + self.next_state.action = ARCHITECTURE_STEP_NAME + return AgentResponse.done(self) + + async def select_templates(self, spec: Specification) -> tuple[str, dict[ProjectTemplateEnum, Any]]: + """ + Select project template(s) to use based on the project description. + + Although the Pythagora database models support multiple projects, this + function will choose at most one project template, as we currently don't + have templates that could be used together in a single project. + + :param spec: Project specification. + :return: Dictionary of selected project templates. + """ + await self.send_message("Selecting starter templates ...") + + llm = self.get_llm() + convo = ( + AgentConvo(self) + .template( + "select_templates", + templates=PROJECT_TEMPLATES, + ) + .require_schema(TemplateSelection) + ) + tpl: TemplateSelection = await llm(convo, parser=JSONParser(TemplateSelection)) + templates = {} + # if tpl.template: + # answer = await self.ask_question( + # f"Do you want to use the '{tpl.template.name}' template?", + # buttons={"yes": "Yes", "no": "No"}, + # default="yes", + # buttons_only=True, + # hint="Project templates are here to speed up start of your app development and save tokens and time.\n" + # "Choose 'Yes' to use suggested template for your app.\n" + # "If you choose 'No', project will be created from scratch.", + # ) + # + # if answer.button == "no": + # return tpl.architecture, templates + # + # template_class = PROJECT_TEMPLATES.get(tpl.template) + # if template_class: + # options = await self.configure_template(spec, template_class) + # templates[tpl.template] = template_class( + # options, + # self.state_manager, + # self.process_manager, + # ) + + return tpl.architecture, templates + + async def plan_architecture(self, spec: Specification): + await self.send_message("Planning project architecture ...") + architecture_description, templates = await self.select_templates(spec) + + await self.send_message("Picking technologies to use ...") + + llm = self.get_llm(stream_output=True) + convo = ( + AgentConvo(self) + .template( + "technologies", + templates=templates, + architecture=architecture_description, + ) + .require_schema(Architecture) + ) + arch: Architecture = await llm(convo, parser=JSONParser(Architecture)) + + await self.check_compatibility(arch) + + spec.architecture = architecture_description + spec.templates = {t.name: t.options_dict for t in templates.values()} + spec.system_dependencies = [d.model_dump() for d in arch.system_dependencies] + spec.package_dependencies = [d.model_dump() for d in arch.package_dependencies] + telemetry.set("architecture", json.loads(arch.model_dump_json())) + + async def check_compatibility(self, arch: Architecture) -> bool: + warn_system_deps = [dep.name for dep in arch.system_dependencies if dep.name.lower() in WARN_SYSTEM_DEPS] + warn_package_deps = [dep.name for dep in arch.package_dependencies if dep.name.lower() in WARN_FRAMEWORKS] + + if warn_system_deps: + await self.ask_question( + f"Warning: Pythagora doesn't officially support {', '.join(warn_system_deps)}. " + f"You can try to use {'it' if len(warn_system_deps) == 1 else 'them'}, but you may run into problems.", + buttons={"continue": "Continue"}, + buttons_only=True, + default="continue", + ) + + if warn_package_deps: + await self.ask_question( + f"Warning: Pythagora works best with vanilla JavaScript. " + f"You can try try to use {', '.join(warn_package_deps)}, but you may run into problems. " + f"Visit {WARN_FRAMEWORKS_URL} for more information.", + buttons={"continue": "Continue"}, + buttons_only=True, + default="continue", + ) + + # TODO: add "cancel" option to the above buttons; if pressed, Architect should + # return AgentResponse.revise_spec() + # that SpecWriter should catch and allow the user to reword the initial spec. + return True + + def prepare_example_project(self, spec: Specification): + log.debug(f"Setting architecture for example project: {spec.example_project}") + arch = EXAMPLE_PROJECTS[spec.example_project]["architecture"] + + spec.architecture = arch["architecture"] + spec.system_dependencies = arch["system_dependencies"] + spec.package_dependencies = arch["package_dependencies"] + spec.templates = arch["templates"] + telemetry.set("templates", spec.templates) + + async def check_system_dependencies(self, spec: Specification): + """ + Check whether the required system dependencies are installed. + + This also stores the app architecture telemetry data, including the + information about whether each system dependency is installed. + + :param spec: Project specification. + """ + deps = spec.system_dependencies + + for dep in deps: + await self.send_message(f"Checking if {dep['name']} is available ...") + status_code, _, _ = await self.process_manager.run_command(dep["test"]) + dep["installed"] = bool(status_code == 0) + if status_code != 0: + if dep["required_locally"]: + remedy = "Please install it before proceeding with your app." + else: + remedy = "If you would like to use it locally, please install it before proceeding." + await self.send_message(f"❌ {dep['name']} is not available. {remedy}") + await self.ask_question( + "", + buttons={"continue": f"I've installed {dep['name']}"}, + buttons_only=True, + default="continue", + ) + + else: + await self.send_message(f"βœ… {dep['name']} is available.") + + async def configure_template(self, spec: Specification, template_class: BaseProjectTemplate) -> BaseModel: + """ + Ask the LLM to configure the template options. + + Based on the project description, the LLM should pick the options that + make the most sense. If template has no options, the method is a no-op + and returns an empty options model. + + :param spec: Project specification. + :param template_class: Template that needs to be configured. + :return: Configured options model. + """ + if template_class.options_class is NoOptions: + # If template has no options, no need to ask LLM for anything + return NoOptions() + + llm = self.get_llm(stream_output=True) + convo = ( + AgentConvo(self) + .template( + "configure_template", + project_description=spec.description, + project_template=template_class, + ) + .require_schema(template_class.options_class) + ) + return await llm(convo, parser=JSONParser(template_class.options_class)) diff --git a/core/agents/base.py b/core/agents/base.py new file mode 100644 index 00000000..7c5f44ff --- /dev/null +++ b/core/agents/base.py @@ -0,0 +1,212 @@ +from typing import Any, Callable, Optional + +from core.agents.response import AgentResponse +from core.config import get_config +from core.db.models import ProjectState +from core.llm.base import BaseLLMClient, LLMError +from core.log import get_logger +from core.proc.process_manager import ProcessManager +from core.state.state_manager import StateManager +from core.ui.base import AgentSource, UIBase, UserInput, pythagora_source + +log = get_logger(__name__) + + +class BaseAgent: + """ + Base class for agents. + """ + + agent_type: str + display_name: str + + def __init__( + self, + state_manager: StateManager, + ui: UIBase, + *, + step: Optional[Any] = None, + prev_response: Optional["AgentResponse"] = None, + process_manager: Optional["ProcessManager"] = None, + data: Optional[Any] = None, + args: Optional[Any] = None, + ): + """ + Create a new agent. + """ + self.ui_source = AgentSource(self.display_name, self.agent_type) + self.ui = ui + self.state_manager = state_manager + self.process_manager = process_manager + self.prev_response = prev_response + self.step = step + self.data = data + self.args = args + + @property + def current_state(self) -> ProjectState: + """Current state of the project (read-only).""" + return self.state_manager.current_state + + @property + def next_state(self) -> ProjectState: + """Next state of the project (write-only).""" + return self.state_manager.next_state + + async def send_message(self, message: str, extra_info: Optional[str] = None): + """ + Send a message to the user. + + Convenience method, uses `UIBase.send_message()` to send the message, + setting the correct source and project state ID. + + :param message: Message to send. + :param extra_info: Extra information to indicate special functionality in extension + """ + await self.ui.send_message( + message + "\n", source=self.ui_source, project_state_id=str(self.current_state.id), extra_info=extra_info + ) + + async def ask_question( + self, + question: str, + *, + buttons: Optional[dict[str, str]] = None, + default: Optional[str] = None, + buttons_only: bool = False, + allow_empty: bool = False, + full_screen: Optional[bool] = False, + hint: Optional[str] = None, + verbose: bool = True, + initial_text: Optional[str] = None, + extra_info: Optional[str] = None, + placeholder: Optional[str] = None, + ) -> UserInput: + """ + Ask a question to the user and return the response. + + Convenience method, uses `UIBase.ask_question()` to + ask the question, setting the correct source and project state ID, and + logging the question/response. + + :param question: Question to ask. + :param buttons: Buttons to display with the question. + :param default: Default button to select. + :param buttons_only: Only display buttons, no text input. + :param allow_empty: Allow empty input. + :param full_screen: Show question full screen in extension. + :param hint: Text to display in a popup as a hint to the question. + :param verbose: Whether to log the question and response. + :param initial_text: Initial text input. + :param extra_info: Extra information to indicate special functionality in extension. + :param placeholder: Placeholder text for the input field. + :return: User response. + """ + response = await self.ui.ask_question( + question, + buttons=buttons, + default=default, + buttons_only=buttons_only, + allow_empty=allow_empty, + full_screen=full_screen, + hint=hint, + verbose=verbose, + initial_text=initial_text, + source=self.ui_source, + project_state_id=str(self.current_state.id), + extra_info=extra_info, + placeholder=placeholder, + ) + await self.state_manager.log_user_input(question, response) + return response + + async def stream_handler(self, content: str): + """ + Handle streamed response from the LLM. + + Serves as a callback to `AgentBase.llm()` so it can stream the responses to the UI. + + :param content: Response content. + """ + + await self.ui.send_stream_chunk(content, source=self.ui_source, project_state_id=str(self.current_state.id)) + + if content is None: + await self.ui.send_message("", source=self.ui_source, project_state_id=str(self.current_state.id)) + + async def error_handler(self, error: LLMError, message: Optional[str] = None) -> bool: + """ + Handle error responses from the LLM. + + :param error: The exception that was thrown the the LLM client. + :param message: Optional message to show. + :return: Whether the request should be retried. + """ + + if error == LLMError.KEY_EXPIRED: + await self.ui.send_key_expired(message) + answer = await self.ask_question( + "Would you like to retry the last step?", + buttons={"yes": "Yes", "no": "No"}, + buttons_only=True, + ) + if answer.button == "yes": + return True + elif error == LLMError.GENERIC_API_ERROR: + await self.stream_handler(message) + answer = await self.ui.ask_question( + "Would you like to retry the failed request?", + buttons={"yes": "Yes", "no": "No"}, + buttons_only=True, + source=pythagora_source, + ) + if answer.button == "yes": + return True + elif error == LLMError.RATE_LIMITED: + await self.stream_handler(message) + + return False + + def get_llm(self, name=None, stream_output=False) -> Callable: + """ + Get a new instance of the agent-specific LLM client. + + The client initializes the UI stream handler and stores the + request/response to the current state's log. The agent name + can be overridden in case the agent needs to use a different + model configuration. + + :param name: Name of the agent for configuration (default: class name). + :return: LLM client for the agent. + """ + + if name is None: + name = self.__class__.__name__ + + config = get_config() + + llm_config = config.llm_for_agent(name) + client_class = BaseLLMClient.for_provider(llm_config.provider) + stream_handler = self.stream_handler if stream_output else None + llm_client = client_class(llm_config, stream_handler=stream_handler, error_handler=self.error_handler) + + async def client(convo, **kwargs) -> Any: + """ + Agent-specific LLM client. + + For details on optional arguments to pass to the LLM client, + see `pythagora.llm.openai_client.OpenAIClient()`. + """ + response, request_log = await llm_client(convo, **kwargs) + await self.state_manager.log_llm_request(request_log, agent=self) + return response + + return client + + async def run() -> AgentResponse: + """ + Run the agent. + + :return: Response from the agent. + """ + raise NotImplementedError() diff --git a/core/agents/bug_hunter.py b/core/agents/bug_hunter.py new file mode 100644 index 00000000..67544658 --- /dev/null +++ b/core/agents/bug_hunter.py @@ -0,0 +1,363 @@ +import json +from enum import Enum + +from pydantic import BaseModel, Field + +from core.agents.base import BaseAgent +from core.agents.convo import AgentConvo +from core.agents.mixins import ChatWithBreakdownMixin, TestSteps +from core.agents.response import AgentResponse +from core.config import CHECK_LOGS_AGENT_NAME, magic_words +from core.db.models.project_state import IterationStatus +from core.llm.parser import JSONParser +from core.log import get_logger +from core.telemetry import telemetry +from core.ui.base import ProjectStage, pythagora_source + +log = get_logger(__name__) + + +class HuntConclusionType(str, Enum): + ADD_LOGS = magic_words.ADD_LOGS + PROBLEM_IDENTIFIED = magic_words.PROBLEM_IDENTIFIED + + +class HuntConclusionOptions(BaseModel): + conclusion: HuntConclusionType = Field( + description=f"If more logs are needed to identify the problem, respond with '{magic_words.ADD_LOGS}'. If the problem is identified, respond with '{magic_words.PROBLEM_IDENTIFIED}'." + ) + + +class ImportantLog(BaseModel): + logCode: str = Field(description="Actual line of code that prints the log.") + shouldBeDifferent: bool = Field( + description="Whether the current output should be different from the expected output." + ) + filePath: str = Field(description="Path to the file in which the log exists.") + currentOutput: str = Field(description="Current output of the log.") + expectedOutput: str = Field(description="Expected output of the log.") + explanation: str = Field(description="A brief explanation of the log.") + + +class ImportantLogsForDebugging(BaseModel): + logs: list[ImportantLog] = Field(description="Important logs that will help the human debug the current bug.") + + +class BugHunter(ChatWithBreakdownMixin, BaseAgent): + agent_type = "bug-hunter" + display_name = "Bug Hunter" + + async def run(self) -> AgentResponse: + current_iteration = self.current_state.current_iteration + + if "bug_reproduction_description" not in current_iteration: + await self.get_bug_reproduction_instructions() + if current_iteration["status"] == IterationStatus.HUNTING_FOR_BUG: + # TODO determine how to find a bug (eg. check in db, ask user a question, etc.) + return await self.check_logs() + elif current_iteration["status"] == IterationStatus.AWAITING_USER_TEST: + await self.ui.send_bug_hunter_status("close_status", 0) + return await self.ask_user_to_test(False, True) + elif current_iteration["status"] == IterationStatus.AWAITING_BUG_REPRODUCTION: + await self.ui.send_bug_hunter_status("close_status", 0) + return await self.ask_user_to_test(True, False) + elif current_iteration["status"] == IterationStatus.START_PAIR_PROGRAMMING: + await self.ui.send_bug_hunter_status("close_status", 0) + return await self.start_pair_programming() + + async def get_bug_reproduction_instructions(self): + await self.send_message("Finding a way to reproduce the bug ...") + llm = self.get_llm() + convo = ( + AgentConvo(self) + .template( + "get_bug_reproduction_instructions", + current_task=self.current_state.current_task, + user_feedback=self.current_state.current_iteration["user_feedback"], + user_feedback_qa=self.current_state.current_iteration["user_feedback_qa"], + docs=self.current_state.docs, + next_solution_to_try=None, + ) + .require_schema(TestSteps) + ) + bug_reproduction_instructions: TestSteps = await llm(convo, parser=JSONParser(TestSteps), temperature=0) + self.next_state.current_iteration["bug_reproduction_description"] = json.dumps( + [test.dict() for test in bug_reproduction_instructions.steps] + ) + + async def check_logs(self, logs_message: str = None): + llm = self.get_llm(CHECK_LOGS_AGENT_NAME, stream_output=True) + convo = self.generate_iteration_convo_so_far() + await self.ui.start_breakdown_stream() + human_readable_instructions = await llm(convo, temperature=0.5) + + convo.assistant(human_readable_instructions) + + human_readable_instructions = await self.chat_with_breakdown(convo, human_readable_instructions) + + convo = ( + AgentConvo(self) + .template( + "bug_found_or_add_logs", + hunt_conclusion=human_readable_instructions, + ) + .require_schema(HuntConclusionOptions) + ) + llm = self.get_llm() + hunt_conclusion = await llm(convo, parser=JSONParser(HuntConclusionOptions), temperature=0) + + bug_hunting_cycles = self.current_state.current_iteration.get("bug_hunting_cycles") + num_bug_hunting_cycles = len(bug_hunting_cycles) if bug_hunting_cycles else 0 + if hunt_conclusion.conclusion == magic_words.PROBLEM_IDENTIFIED: + # if no need for logs, implement iteration same as before + self.set_data_for_next_hunting_cycle(human_readable_instructions, IterationStatus.AWAITING_BUG_FIX) + await self.send_message("Found the bug. I'm attempting to fix it ...") + await self.ui.send_bug_hunter_status("fixing_bug", num_bug_hunting_cycles) + else: + # if logs are needed, add logging steps + self.set_data_for_next_hunting_cycle(human_readable_instructions, IterationStatus.AWAITING_LOGGING) + await self.send_message("Adding more logs to identify the bug ...") + await self.ui.send_bug_hunter_status("adding_logs", num_bug_hunting_cycles) + + self.next_state.flag_iterations_as_modified() + return AgentResponse.done(self) + + async def ask_user_to_test(self, awaiting_bug_reproduction: bool = False, awaiting_user_test: bool = False): + await self.ui.stop_app() + test_instructions = self.current_state.current_iteration["bug_reproduction_description"] + await self.ui.send_message( + "Start the app and test it by following these instructions:\n\n", source=pythagora_source + ) + await self.send_message("") + await self.ui.send_test_instructions(test_instructions, project_state_id=str(self.current_state.id)) + + if self.current_state.run_command: + await self.ui.send_run_command(self.current_state.run_command) + + await self.ask_question( + "Please test the app again.", + buttons={"done": "I am done testing"}, + buttons_only=True, + default="continue", + extra_info="restart_app", + hint="Instructions for testing:\n\n" + self.current_state.current_iteration["bug_reproduction_description"], + ) + + if awaiting_user_test: + buttons = {"yes": "Yes, the issue is fixed", "no": "No", "start_pair_programming": "Start Pair Programming"} + user_feedback = await self.ask_question( + "Is the bug you reported fixed now?", + buttons=buttons, + default="yes", + buttons_only=True, + hint="Instructions for testing:\n\n" + + self.current_state.current_iteration["bug_reproduction_description"], + ) + self.next_state.current_iteration["bug_hunting_cycles"][-1]["fix_attempted"] = True + + if user_feedback.button == "yes": + self.next_state.complete_iteration() + elif user_feedback.button == "start_pair_programming": + self.next_state.current_iteration["status"] = IterationStatus.START_PAIR_PROGRAMMING + self.next_state.flag_iterations_as_modified() + else: + awaiting_bug_reproduction = True + + if awaiting_bug_reproduction: + buttons = { + "done": "Bug is fixed", + "continue": "Continue without feedback", # DO NOT CHANGE THIS TEXT without changing it in the extension (it is hardcoded) + "start_pair_programming": "Start Pair Programming", + } + await self.ui.send_project_stage( + { + "stage": ProjectStage.ADDITIONAL_FEEDBACK, + } + ) + user_feedback = await self.ask_question( + "Please add any additional feedback that could help Pythagora solve this bug", + buttons=buttons, + default="continue", + extra_info="collect_logs", + hint="Instructions for testing:\n\n" + + self.current_state.current_iteration["bug_reproduction_description"], + ) + + if user_feedback.button == "done": + self.next_state.complete_iteration() + return AgentResponse.done(self) + elif user_feedback.button == "start_pair_programming": + self.next_state.current_iteration["status"] = IterationStatus.START_PAIR_PROGRAMMING + self.next_state.flag_iterations_as_modified() + return AgentResponse.done(self) + + # TODO select only the logs that are new (with PYTHAGORA_DEBUGGING_LOG) + self.next_state.current_iteration["bug_hunting_cycles"][-1]["backend_logs"] = None + self.next_state.current_iteration["bug_hunting_cycles"][-1]["frontend_logs"] = None + self.next_state.current_iteration["bug_hunting_cycles"][-1]["user_feedback"] = user_feedback.text + self.next_state.current_iteration["status"] = IterationStatus.HUNTING_FOR_BUG + + return AgentResponse.done(self) + + async def start_pair_programming(self): + llm = self.get_llm(stream_output=True) + convo = self.generate_iteration_convo_so_far(True) + if len(convo.messages) > 1: + convo.remove_last_x_messages(1) + convo = convo.template("problem_explanation") + await self.ui.start_important_stream() + initial_explanation = await llm(convo, temperature=0.5) + + llm = self.get_llm() + convo = convo.template("data_about_logs").require_schema(ImportantLogsForDebugging) + data_about_logs = await llm(convo, parser=JSONParser(ImportantLogsForDebugging), temperature=0.5) + + await self.ui.send_data_about_logs( + { + "logs": [ + { + "currentLog": d.currentOutput, + "expectedLog": d.expectedOutput, + "explanation": d.explanation, + "filePath": d.filePath, + "logCode": d.logCode, + "shouldBeDifferent": d.shouldBeDifferent, + } + for d in data_about_logs.logs + ] + } + ) + + while True: + self.next_state.current_iteration["initial_explanation"] = initial_explanation + next_step = await self.ask_question( + "What do you want to do?", + buttons={ + "question": "I have a question", + "done": "I fixed the bug myself", + "tell_me_more": "Tell me more about the bug", + "solution_hint": "I think I know where the problem is", + "other": "Other", + }, + buttons_only=True, + default="continue", + hint="Instructions for testing:\n\n" + + self.current_state.current_iteration["bug_reproduction_description"], + ) + + await telemetry.trace_code_event( + "pair-programming", + { + "button": next_step.button, + "num_tasks": len(self.current_state.tasks), + "num_epics": len(self.current_state.epics), + "num_iterations": len(self.current_state.iterations), + "app_id": str(self.state_manager.project.id), + "app_name": self.state_manager.project.name, + "folder_name": self.state_manager.project.folder_name, + }, + ) + + # TODO: remove when Leon checks + convo.remove_last_x_messages(2) + + if len(convo.messages) > 10: + convo.trim(1, 2) + + # TODO: in the future improve with a separate conversation that parses the user info and goes into an appropriate if statement + if next_step.button == "done": + self.next_state.complete_iteration() + break + elif next_step.button == "question": + user_response = await self.ask_question("Oh, cool, what would you like to know?") + convo = convo.template("ask_a_question", question=user_response.text) + await self.ui.start_important_stream() + llm_answer = await llm(convo, temperature=0.5) + await self.send_message(llm_answer) + elif next_step.button == "tell_me_more": + convo.template("tell_me_more") + await self.ui.start_important_stream() + response = await llm(convo, temperature=0.5) + await self.send_message(response) + elif next_step.button == "other": + # this is the same as "question" - we want to keep an option for users to click to understand if we're missing something with other options + user_response = await self.ask_question("Let me know what you think ...") + convo = convo.template("ask_a_question", question=user_response.text) + await self.ui.start_important_stream() + llm_answer = await llm(convo, temperature=0.5) + await self.send_message(llm_answer) + elif next_step.button == "solution_hint": + human_hint_label = "Amazing! How do you think we can solve this bug?" + while True: + human_hint = await self.ask_question(human_hint_label) + convo = convo.template("instructions_from_human_hint", human_hint=human_hint.text) + await self.ui.start_important_stream() + llm = self.get_llm(CHECK_LOGS_AGENT_NAME, stream_output=True) + human_readable_instructions = await llm(convo, temperature=0.5) + human_approval = await self.ask_question( + "Can I implement this solution?", buttons={"yes": "Yes", "no": "No"}, buttons_only=True + ) + llm = self.get_llm(stream_output=True) + if human_approval.button == "yes": + self.set_data_for_next_hunting_cycle( + human_readable_instructions, IterationStatus.AWAITING_BUG_FIX + ) + self.next_state.flag_iterations_as_modified() + break + else: + human_hint_label = "Oh, my bad, what did I misunderstand?" + break + elif next_step.button == "tell_me_more": + convo.template("tell_me_more") + await self.ui.start_important_stream() + response = await llm(convo, temperature=0.5) + await self.send_message(response) + continue + + return AgentResponse.done(self) + + def generate_iteration_convo_so_far(self, omit_last_cycle=False): + convo = AgentConvo(self).template( + "iteration", + current_task=self.current_state.current_task, + user_feedback=self.current_state.current_iteration["user_feedback"], + user_feedback_qa=self.current_state.current_iteration["user_feedback_qa"], + docs=self.current_state.docs, + magic_words=magic_words, + next_solution_to_try=None, + test_instructions=json.loads(self.current_state.current_task.get("test_instructions") or "[]"), + ) + + hunting_cycles = self.current_state.current_iteration.get("bug_hunting_cycles", [])[ + 0 : (-1 if omit_last_cycle else None) + ] + + for hunting_cycle in hunting_cycles: + convo = convo.assistant(hunting_cycle["human_readable_instructions"]).template( + "log_data", + backend_logs=hunting_cycle.get("backend_logs"), + frontend_logs=hunting_cycle.get("frontend_logs"), + fix_attempted=hunting_cycle.get("fix_attempted"), + user_feedback=hunting_cycle.get("user_feedback"), + ) + + return convo + + def set_data_for_next_hunting_cycle(self, human_readable_instructions, new_status): + self.next_state.current_iteration["description"] = human_readable_instructions + self.next_state.current_iteration["bug_hunting_cycles"] += [ + { + "human_readable_instructions": human_readable_instructions, + "fix_attempted": any( + c["fix_attempted"] for c in self.current_state.current_iteration["bug_hunting_cycles"] + ), + } + ] + + self.next_state.current_iteration["status"] = new_status + + async def continue_on(self, convo, button_value, user_response): + llm = self.get_llm(stream_output=True) + convo = convo.template("continue_on") + continue_on = await llm(convo, temperature=0.5) + return continue_on diff --git a/core/agents/code_monkey.py b/core/agents/code_monkey.py new file mode 100644 index 00000000..0625b8de --- /dev/null +++ b/core/agents/code_monkey.py @@ -0,0 +1,460 @@ +import re +from difflib import unified_diff +from enum import Enum +from typing import Optional, Union + +from pydantic import BaseModel, Field + +from core.agents.base import BaseAgent +from core.agents.convo import AgentConvo +from core.agents.mixins import FileDiffMixin +from core.agents.response import AgentResponse, ResponseType +from core.config import CODE_MONKEY_AGENT_NAME, CODE_REVIEW_AGENT_NAME, DESCRIBE_FILES_AGENT_NAME +from core.llm.parser import JSONParser, OptionalCodeBlockParser +from core.log import get_logger + +log = get_logger(__name__) + + +# Constant for indicating missing new line at the end of a file in a unified diff +NO_EOL = "\\ No newline at end of file" + +# Regular expression pattern for matching hunk headers +PATCH_HEADER_PATTERN = re.compile(r"^@@ -(\d+),?(\d+)? \+(\d+),?(\d+)? @@") + +# Maximum number of attempts to ask for review if it can't be parsed +MAX_REVIEW_RETRIES = 2 + +# Maximum number of code implementation attempts after which we accept the changes unconditionaly +MAX_CODING_ATTEMPTS = 3 + + +class Decision(str, Enum): + APPLY = "apply" + IGNORE = "ignore" + REWORK = "rework" + + +class Hunk(BaseModel): + number: int = Field(description="Index of the hunk in the diff. Starts from 1.") + reason: str = Field(description="Reason for applying or ignoring this hunk, or for asking for it to be reworked.") + decision: Decision = Field(description="Whether to apply this hunk, rework, or ignore it.") + + +class ReviewChanges(BaseModel): + hunks: list[Hunk] + review_notes: str = Field(description="Additional review notes (optional, can be empty).") + + +class FileDescription(BaseModel): + summary: str = Field( + description="Detailed description summarized what the file is about, and what the major classes, functions, elements or other functionality is implemented." + ) + references: list[str] = Field( + description="List of references the file imports or includes (only files local to the project), where each element specifies the project-relative path of the referenced file, including the file extension." + ) + + +class CodeMonkey(FileDiffMixin, BaseAgent): + agent_type = "code-monkey" + display_name = "Code Monkey" + + async def run(self) -> AgentResponse: + if self.prev_response and self.prev_response.type == ResponseType.DESCRIBE_FILES: + return await self.describe_files() + else: + data = await self.implement_changes() + code_review_done = False + while not code_review_done: + review_response = await self.run_code_review(data) + if isinstance(review_response, AgentResponse): + return review_response + data = await self.implement_changes(review_response) + + async def implement_changes(self, data: Optional[dict] = None) -> dict: + file_name = self.step["save_file"]["path"] + + current_file = await self.state_manager.get_file_by_path(file_name) + file_content = current_file.content.content if current_file else "" + + task = self.current_state.current_task + + if data is not None: + attempt = data["attempt"] + 1 + feedback = data["feedback"] + log.debug(f"Fixing file {file_name} after review feedback: {feedback} ({attempt}. attempt)") + await self.ui.send_file_status(file_name, "reworking", source=self.ui_source) + else: + log.debug(f"Implementing file {file_name}") + if data is None: + await self.ui.send_file_status( + file_name, "updating" if file_content else "creating", source=self.ui_source + ) + else: + await self.ui.send_file_status(file_name, "reworking", source=self.ui_source) + self.next_state.action = "Updating files" + attempt = 1 + feedback = None + + iterations = self.current_state.iterations + user_feedback = None + user_feedback_qa = None + llm = self.get_llm(CODE_MONKEY_AGENT_NAME) + + if iterations: + last_iteration = iterations[-1] + instructions = last_iteration.get("description") + user_feedback = last_iteration.get("user_feedback") + user_feedback_qa = last_iteration.get("user_feedback_qa") + else: + instructions = self.current_state.current_task["instructions"] + + convo = AgentConvo(self).template( + "implement_changes", + file_name=file_name, + file_content=file_content, + instructions=instructions, + user_feedback=user_feedback, + user_feedback_qa=user_feedback_qa, + ) + if feedback: + convo.assistant(f"```\n{data['new_content']}\n```\n").template( + "review_feedback", + content=data["approved_content"], + original_content=file_content, + rework_feedback=feedback, + ) + + response: str = await llm(convo, temperature=0, parser=OptionalCodeBlockParser()) + # FIXME: provide a counter here so that we don't have an endless loop here + return { + "path": file_name, + "instructions": task["instructions"], + "old_content": file_content, + "new_content": response, + "attempt": attempt, + } + + async def describe_files(self) -> AgentResponse: + llm = self.get_llm(DESCRIBE_FILES_AGENT_NAME) + to_describe = { + file.path: file.content.content for file in self.current_state.files if not file.meta.get("description") + } + + for file in self.next_state.files: + content = to_describe.get(file.path) + if content is None: + continue + + if content == "": + file.meta = { + **file.meta, + "description": "Empty file", + "references": [], + } + continue + + log.debug(f"Describing file {file.path}") + convo = ( + AgentConvo(self) + .template( + "describe_file", + path=file.path, + content=content, + ) + .require_schema(FileDescription) + ) + llm_response: FileDescription = await llm(convo, parser=JSONParser(spec=FileDescription)) + + file.meta = { + **file.meta, + "description": llm_response.summary, + "references": llm_response.references, + } + return AgentResponse.done(self) + + # ------------------------------ + # CODE REVIEW + # ------------------------------ + + async def run_code_review(self, data: Optional[dict]) -> Union[AgentResponse, dict]: + await self.ui.send_file_status(data["path"], "reviewing", source=self.ui_source) + if ( + data is not None + and not data["old_content"] + or data["new_content"] == data["old_content"] + or data["attempt"] >= MAX_CODING_ATTEMPTS + ): + # we always auto-accept new files and unchanged files, or if we've tried too many times + return await self.accept_changes(data["path"], data["old_content"], data["new_content"]) + + approved_content, feedback = await self.review_change( + data["path"], + data["instructions"], + data["old_content"], + data["new_content"], + ) + if feedback: + return { + "new_content": data["new_content"], + "approved_content": approved_content, + "feedback": feedback, + "attempt": data["attempt"], + } + else: + return await self.accept_changes(data["path"], data["old_content"], approved_content) + + async def accept_changes(self, file_path: str, old_content: str, new_content: str) -> AgentResponse: + await self.ui.send_file_status(file_path, "done", source=self.ui_source) + + n_new_lines, n_del_lines = self.get_line_changes(old_content, new_content) + await self.ui.generate_diff( + file_path, old_content, new_content, n_new_lines, n_del_lines, source=self.ui_source + ) + + await self.state_manager.save_file(file_path, new_content) + self.next_state.complete_step("save_file") + + input_required = self.state_manager.get_input_required(new_content, file_path) + if input_required: + return AgentResponse.input_required( + self, + [{"file": file_path, "line": line} for line in input_required], + ) + else: + return AgentResponse.done(self) + + def _get_task_convo(self) -> AgentConvo: + # FIXME: Current prompts reuse conversation from the developer so we have to resort to this + task = self.current_state.current_task + current_task_index = self.current_state.tasks.index(task) + + convo = AgentConvo(self).template( + "breakdown", + task=task, + iteration=None, + current_task_index=current_task_index, + related_api_endpoints=task.get("related_api_endpoints", []), + ) + # TODO: We currently show last iteration to the code monkey; we might need to show the task + # breakdown and all the iterations instead? To think about when refactoring prompts + if self.current_state.iterations: + convo.assistant(self.current_state.iterations[-1]["description"]) + else: + convo.assistant(self.current_state.current_task["instructions"]) + return convo + + async def review_change( + self, file_name: str, instructions: str, old_content: str, new_content: str + ) -> tuple[str, str]: + """ + Review changes that were applied to the file. + + This asks the LLM to act as a PR reviewer and for each part (hunk) of the + diff, decide if it should be applied (kept) or ignored (removed from the PR). + + :param file_name: name of the file being modified + :param instructions: instructions for the reviewer + :param old_content: old file content + :param new_content: new file content (with proposed changes) + :return: tuple with file content update with approved changes, and review feedback + + Diff hunk explanation: https://www.gnu.org/software/diffutils/manual/html_node/Hunks.html + """ + + hunks = self.get_diff_hunks(file_name, old_content, new_content) + + llm = self.get_llm(CODE_REVIEW_AGENT_NAME) + convo = ( + self._get_task_convo() + .template( + "review_changes", + instructions=instructions, + file_name=file_name, + old_content=old_content, + hunks=hunks, + ) + .require_schema(ReviewChanges) + ) + llm_response: ReviewChanges = await llm(convo, temperature=0, parser=JSONParser(ReviewChanges)) + + for i in range(MAX_REVIEW_RETRIES): + reasons = {} + ids_to_apply = set() + ids_to_ignore = set() + ids_to_rework = set() + for hunk in llm_response.hunks: + reasons[hunk.number - 1] = hunk.reason + if hunk.decision == "apply": + ids_to_apply.add(hunk.number - 1) + elif hunk.decision == "ignore": + ids_to_ignore.add(hunk.number - 1) + elif hunk.decision == "rework": + ids_to_rework.add(hunk.number - 1) + + n_hunks = len(hunks) + n_review_hunks = len(reasons) + if n_review_hunks == n_hunks: + break + elif n_review_hunks < n_hunks: + error = "Not all hunks have been reviewed. Please review all hunks and add 'apply', 'ignore' or 'rework' decision for each." + elif n_review_hunks > n_hunks: + error = f"Your review contains more hunks ({n_review_hunks}) than in the original diff ({n_hunks}). Note that one hunk may have multiple changed lines." + + # Max two retries; if the reviewer still hasn't reviewed all hunks, we'll just use the entire new content + convo.assistant(llm_response.model_dump_json()).user(error) + llm_response = await llm(convo, parser=JSONParser(ReviewChanges)) + else: + return new_content, None + + hunks_to_apply = [h for i, h in enumerate(hunks) if i in ids_to_apply] + diff_log = f"--- {file_name}\n+++ {file_name}\n" + "\n".join(hunks_to_apply) + + hunks_to_rework = [(i, h) for i, h in enumerate(hunks) if i in ids_to_rework] + review_log = ( + "\n\n".join([f"## Change\n```{hunk}```\nReviewer feedback:\n{reasons[i]}" for (i, hunk) in hunks_to_rework]) + + "\n\nReview notes:\n" + + llm_response.review_notes + ) + + if len(hunks_to_apply) == len(hunks): + log.info(f"Applying entire change to {file_name}") + return new_content, None + + elif len(hunks_to_apply) == 0: + if hunks_to_rework: + log.info(f"Requesting rework for {len(hunks_to_rework)} changes to {file_name} (0 hunks to apply)") + return old_content, review_log + else: + # If everything can be safely ignored, it's probably because the files already implement the changes + # from previous tasks (which can happen often). Insisting on a change here is likely to cause problems. + log.info(f"Rejecting entire change to {file_name} with reason: {llm_response.review_notes}") + return old_content, None + + log.debug(f"Applying code change to {file_name}:\n{diff_log}") + new_content = self.apply_diff(file_name, old_content, hunks_to_apply, new_content) + if hunks_to_rework: + log.info(f"Requesting further rework for {len(hunks_to_rework)} changes to {file_name}") + return new_content, review_log + else: + return new_content, None + + @staticmethod + def get_diff_hunks(file_name: str, old_content: str, new_content: str) -> list[str]: + """ + Get the diff between two files. + + This uses Python difflib to produce an unified diff, then splits + it into hunks that will be separately reviewed by the reviewer. + + :param file_name: name of the file being modified + :param old_content: old file content + :param new_content: new file content + :return: change hunks from the unified diff + """ + from_name = "old_" + file_name + to_name = "to_" + file_name + from_lines = old_content.splitlines(keepends=True) + to_lines = new_content.splitlines(keepends=True) + diff_gen = unified_diff(from_lines, to_lines, fromfile=from_name, tofile=to_name) + diff_txt = "".join(diff_gen) + + hunks = re.split(r"\n@@", diff_txt, re.MULTILINE) + result = [] + for i, h in enumerate(hunks): + # Skip the prologue (file names) + if i == 0: + continue + txt = h.splitlines() + txt[0] = "@@" + txt[0] + result.append("\n".join(txt)) + return result + + def apply_diff(self, file_name: str, old_content: str, hunks: list[str], fallback: str): + """ + Apply the diff to the original file content. + + This uses the internal `_apply_patch` method to apply the + approved diff hunks to the original file content. + + If patch apply fails, the fallback is the full new file content + with all the changes applied (as if the reviewer approved everythng). + + :param file_name: name of the file being modified + :param old_content: old file content + :param hunks: change hunks from the unified diff + :param fallback: proposed new file content (with all the changes applied) + """ + diff = ( + "\n".join( + [ + f"--- {file_name}", + f"+++ {file_name}", + ] + + hunks + ) + + "\n" + ) + try: + fixed_content = self._apply_patch(old_content, diff) + except Exception as e: + # This should never happen but if it does, just use the new version from + # the LLM and hope for the best + print(f"Error applying diff: {e}; hoping all changes are valid") + return fallback + + return fixed_content + + # Adapted from https://gist.github.com/noporpoise/16e731849eb1231e86d78f9dfeca3abc (Public Domain) + @staticmethod + def _apply_patch(original: str, patch: str, revert: bool = False): + """ + Apply a patch to a string to recover a newer version of the string. + + :param original: The original string. + :param patch: The patch to apply. + :param revert: If True, treat the original string as the newer version and recover the older string. + :return: The updated string after applying the patch. + """ + original_lines = original.splitlines(True) + patch_lines = patch.splitlines(True) + + updated_text = "" + index_original = start_line = 0 + + # Choose which group of the regex to use based on the revert flag + match_index, line_sign = (1, "+") if not revert else (3, "-") + + # Skip header lines of the patch + while index_original < len(patch_lines) and patch_lines[index_original].startswith(("---", "+++")): + index_original += 1 + + while index_original < len(patch_lines): + match = PATCH_HEADER_PATTERN.match(patch_lines[index_original]) + if not match: + raise Exception("Bad patch -- regex mismatch [line " + str(index_original) + "]") + + line_number = int(match.group(match_index)) - 1 + (match.group(match_index + 1) == "0") + + if start_line > line_number or line_number > len(original_lines): + raise Exception("Bad patch -- bad line number [line " + str(index_original) + "]") + + updated_text += "".join(original_lines[start_line:line_number]) + start_line = line_number + index_original += 1 + + while index_original < len(patch_lines) and patch_lines[index_original][0] != "@": + if index_original + 1 < len(patch_lines) and patch_lines[index_original + 1][0] == "\\": + line_content = patch_lines[index_original][:-1] + index_original += 2 + else: + line_content = patch_lines[index_original] + index_original += 1 + + if line_content: + if line_content[0] == line_sign or line_content[0] == " ": + updated_text += line_content[1:] + start_line += line_content[0] != line_sign + + updated_text += "".join(original_lines[start_line:]) + return updated_text diff --git a/core/agents/convo.py b/core/agents/convo.py new file mode 100644 index 00000000..60c05f24 --- /dev/null +++ b/core/agents/convo.py @@ -0,0 +1,123 @@ +import json +import sys +from copy import deepcopy +from typing import TYPE_CHECKING, Optional + +import jsonref +from pydantic import BaseModel + +from core.config import get_config +from core.llm.convo import Convo +from core.llm.prompt import JinjaFileTemplate +from core.log import get_logger + +if TYPE_CHECKING: + from core.agents.response import BaseAgent + +log = get_logger(__name__) + + +class AgentConvo(Convo): + prompt_loader: Optional[JinjaFileTemplate] = None + + def __init__(self, agent: "BaseAgent"): + self.agent_instance = agent + + super().__init__() + try: + system_message = self.render("system") + self.system(system_message) + except ValueError as err: + log.warning(f"Agent {agent.__class__.__name__} has no system prompt: {err}") + + @classmethod + def _init_templates(cls): + if cls.prompt_loader is not None: + return + + config = get_config() + cls.prompt_loader = JinjaFileTemplate(config.prompt.paths) + + def _get_default_template_vars(self) -> dict: + if sys.platform == "win32": + os = "Windows" + elif sys.platform == "darwin": + os = "macOS" + else: + os = "Linux" + + return { + "state": self.agent_instance.current_state, + "os": os, + } + + @staticmethod + def _serialize_prompt_context(context: dict) -> dict: + """ + Convert data to JSON serializable format + + This is done by replacing non-serializable values with + their string representations. This is useful for logging. + """ + return json.loads(json.dumps(context, default=lambda o: str(o))) + + def render(self, name: str, **kwargs) -> str: + self._init_templates() + + kwargs.update(self._get_default_template_vars()) + + # Jinja uses "/" even in Windows + template_name = f"{self.agent_instance.agent_type}/{name}.prompt" + log.debug(f"Loading template {template_name}") + return self.prompt_loader(template_name, **kwargs) + + def template(self, template_name: str, **kwargs) -> "AgentConvo": + message = self.render(template_name, **kwargs) + self.user(message) + self.prompt_log.append( + { + "template": f"{self.agent_instance.agent_type}/{template_name}", + "context": self._serialize_prompt_context(kwargs), + } + ) + return self + + def fork(self) -> "AgentConvo": + child = AgentConvo(self.agent_instance) + child.messages = deepcopy(self.messages) + child.prompt_log = deepcopy(self.prompt_log) + return child + + def trim(self, trim_index: int, trim_count: int) -> "AgentConvo": + """ + Trim the conversation starting from the given index by 1 message. + :param trim_index: + :return: + """ + self.messages = self.messages[:trim_index] + self.messages[trim_index + trim_count :] + return self + + def require_schema(self, model: BaseModel) -> "AgentConvo": + def remove_defs(d): + if isinstance(d, dict): + return {k: remove_defs(v) for k, v in d.items() if k != "$defs"} + elif isinstance(d, list): + return [remove_defs(v) for v in d] + else: + return d + + # We want to make the schema as simple as possible to avoid confusing the LLM, + # so we remove (dereference) all the refs we can and show the "final" schema version. + schema_txt = json.dumps(remove_defs(jsonref.loads(json.dumps(model.model_json_schema())))) + self.user( + f"IMPORTANT: Your response MUST conform to this JSON schema:\n```\n{schema_txt}\n```." + f"YOU MUST NEVER add any additional fields to your response, and NEVER add additional preamble like 'Here is your JSON'." + ) + return self + + def remove_last_x_messages(self, x: int) -> "AgentConvo": + """ + Remove the last `x` messages from the conversation. + """ + self.messages = self.messages[:-x] + return self diff --git a/core/agents/developer.py b/core/agents/developer.py new file mode 100644 index 00000000..ad1b6819 --- /dev/null +++ b/core/agents/developer.py @@ -0,0 +1,366 @@ +import json +from enum import Enum +from typing import Annotated, Literal, Union +from uuid import uuid4 + +from pydantic import BaseModel, Field + +from core.agents.base import BaseAgent +from core.agents.convo import AgentConvo +from core.agents.mixins import ChatWithBreakdownMixin, RelevantFilesMixin +from core.agents.response import AgentResponse +from core.config import PARSE_TASK_AGENT_NAME, TASK_BREAKDOWN_AGENT_NAME +from core.db.models.project_state import IterationStatus, TaskStatus +from core.db.models.specification import Complexity +from core.llm.parser import JSONParser +from core.log import get_logger +from core.telemetry import telemetry +from core.ui.base import ProjectStage + +log = get_logger(__name__) + + +class StepType(str, Enum): + COMMAND = "command" + SAVE_FILE = "save_file" + HUMAN_INTERVENTION = "human_intervention" + UTILITY_FUNCTION = "utility_function" + + +class CommandOptions(BaseModel): + command: str = Field(description="Command to run") + timeout: int = Field(description="Timeout in seconds") + success_message: str = "" + + +class SaveFileOptions(BaseModel): + path: str + + +class SaveFileStep(BaseModel): + type: Literal[StepType.SAVE_FILE] = StepType.SAVE_FILE + save_file: SaveFileOptions + related_api_endpoints: list[str] = Field(description="API endpoints that are implemented in this file", default=[]) + + +class CommandStep(BaseModel): + type: Literal[StepType.COMMAND] = StepType.COMMAND + command: CommandOptions + + +class HumanInterventionStep(BaseModel): + type: Literal[StepType.HUMAN_INTERVENTION] = StepType.HUMAN_INTERVENTION + human_intervention_description: str + + +class UtilityFunction(BaseModel): + type: Literal[StepType.UTILITY_FUNCTION] = StepType.UTILITY_FUNCTION + file: str + function_name: str + description: str + return_value: str + input_value: str + status: Literal["mocked", "implemented"] + + +Step = Annotated[ + Union[SaveFileStep, CommandStep, HumanInterventionStep, UtilityFunction], + Field(discriminator="type"), +] + + +class TaskSteps(BaseModel): + steps: list[Step] + + +class Developer(ChatWithBreakdownMixin, RelevantFilesMixin, BaseAgent): + agent_type = "developer" + display_name = "Developer" + + async def run(self) -> AgentResponse: + if self.current_state.current_step and self.current_state.current_step.get("type") == "utility_function": + return await self.update_knowledge_base() + + if not self.current_state.unfinished_tasks: + log.warning("No unfinished tasks found, nothing to do (why am I called? is this a bug?)") + return AgentResponse.done(self) + + if self.current_state.unfinished_iterations: + return await self.breakdown_current_iteration() + + # By default, we want to ask the user if they want to run the task, + # except in certain cases (such as they've just edited it). + # The check for docs is here to prevent us from asking the user whether we should + # run the task twice - we'll only ask if we haven't yet checked for docs. + if not self.current_state.current_task.get("run_always", False) and self.current_state.docs is None: + if not await self.ask_to_execute_task(): + return AgentResponse.done(self) + + if self.current_state.docs is None and self.current_state.specification.complexity != Complexity.SIMPLE: + # We check for external docs here, to make sure we only fetch the docs + # if the task is actually being done. + return AgentResponse.external_docs_required(self) + + return await self.breakdown_current_task() + + async def breakdown_current_iteration(self) -> AgentResponse: + """ + Breaks down current iteration or task review into steps. + + :return: AgentResponse.done(self) when the breakdown is done + """ + current_task = self.current_state.current_task + + if self.current_state.current_iteration["status"] in ( + IterationStatus.AWAITING_BUG_FIX, + IterationStatus.AWAITING_LOGGING, + ): + iteration = self.current_state.current_iteration + + description = iteration["bug_hunting_cycles"][-1]["human_readable_instructions"] + user_feedback = iteration["user_feedback"] + source = "bug_hunt" + n_tasks = len(self.next_state.iterations) + log.debug(f"Breaking down the logging cycle {description}") + else: + iteration = self.current_state.current_iteration + if iteration is None: + log.error("Iteration breakdown called but there's no current iteration or task review, possible bug?") + return AgentResponse.done(self) + + description = iteration["description"] + user_feedback = iteration["user_feedback"] + source = "troubleshooting" + n_tasks = len(self.next_state.iterations) + log.debug(f"Breaking down the iteration {description}") + + if self.current_state.files and self.current_state.relevant_files is None: + await self.get_relevant_files(user_feedback, description) + + await self.ui.send_task_progress( + n_tasks, # iterations and reviews can be created only one at a time, so we are always on last one + n_tasks, + current_task["description"], + source, + "in-progress", + self.current_state.get_source_index(source), + self.current_state.tasks, + ) + llm = self.get_llm(PARSE_TASK_AGENT_NAME) + # FIXME: In case of iteration, parse_task depends on the context (files, tasks, etc) set there. + # Ideally this prompt would be self-contained. + convo = ( + AgentConvo(self) + .template( + "iteration", + user_feedback=user_feedback, + user_feedback_qa=None, + next_solution_to_try=None, + docs=self.current_state.docs, + test_instructions=json.loads(current_task.get("test_instructions") or "[]"), + ) + .assistant(description) + .template("parse_task") + .require_schema(TaskSteps) + ) + response: TaskSteps = await llm(convo, parser=JSONParser(TaskSteps), temperature=0) + + self.set_next_steps(response, source) + + if iteration: + if "status" not in iteration or ( + iteration["status"] in (IterationStatus.AWAITING_USER_TEST, IterationStatus.AWAITING_BUG_REPRODUCTION) + ): + # This is just a support for old iterations that don't have status + self.next_state.complete_iteration() + self.next_state.action = f"Troubleshooting #{len(self.current_state.iterations)}" + elif iteration["status"] == IterationStatus.IMPLEMENT_SOLUTION: + # If the user requested a change, then, we'll implement it and go straight back to testing + self.next_state.complete_iteration() + self.next_state.action = f"Troubleshooting #{len(self.current_state.iterations)}" + elif iteration["status"] == IterationStatus.AWAITING_BUG_FIX: + # If bug fixing is done, ask user to test again + self.next_state.current_iteration["status"] = IterationStatus.AWAITING_USER_TEST + elif iteration["status"] == IterationStatus.AWAITING_LOGGING: + # If logging is done, ask user to reproduce the bug + self.next_state.current_iteration["status"] = IterationStatus.AWAITING_BUG_REPRODUCTION + else: + self.next_state.action = "Task review feedback" + + current_task_index = self.current_state.tasks.index(current_task) + self.next_state.tasks[current_task_index] = { + **current_task, + } + self.next_state.flag_tasks_as_modified() + return AgentResponse.done(self) + + async def breakdown_current_task(self) -> AgentResponse: + current_task = self.current_state.current_task + source = self.current_state.current_epic.get("source", "app") + await self.ui.send_task_progress( + self.current_state.tasks.index(current_task) + 1, + len(self.current_state.tasks), + current_task["description"], + source, + "in-progress", + self.current_state.get_source_index(source), + self.current_state.tasks, + ) + + log.debug(f"Breaking down the current task: {current_task['description']}") + + log.debug(f"Current state files: {len(self.current_state.files)}, relevant {self.current_state.relevant_files}") + # Check which files are relevant to the current task + await self.get_relevant_files() + + current_task_index = self.current_state.tasks.index(current_task) + + await self.send_message("Thinking about how to implement this task ...") + + await self.ui.start_breakdown_stream() + related_api_endpoints = current_task.get("related_api_endpoints", []) + llm = self.get_llm(TASK_BREAKDOWN_AGENT_NAME, stream_output=True) + convo = AgentConvo(self).template( + "breakdown", + task=current_task, + iteration=None, + current_task_index=current_task_index, + docs=self.current_state.docs, + related_api_endpoints=related_api_endpoints, + ) + response: str = await llm(convo) + convo.assistant(response) + + response = await self.chat_with_breakdown(convo, response) + + self.next_state.tasks[current_task_index] = { + **current_task, + "instructions": response, + } + self.next_state.flag_tasks_as_modified() + + llm = self.get_llm(PARSE_TASK_AGENT_NAME) + convo.template("parse_task").require_schema(TaskSteps) + response: TaskSteps = await llm(convo, parser=JSONParser(TaskSteps), temperature=0) + + # There might be state leftovers from previous tasks that we need to clean here + self.next_state.modified_files = {} + self.set_next_steps(response, source) + self.next_state.action = f"Task #{current_task_index + 1} start" + await telemetry.trace_code_event( + "task-start", + { + "task_index": current_task_index + 1, + "num_tasks": len(self.current_state.tasks), + "num_epics": len(self.current_state.epics), + }, + ) + return AgentResponse.done(self) + + def set_next_steps(self, response: TaskSteps, source: str): + # For logging/debugging purposes, we don't want to remove the finished steps + # until we're done with the task. + unique_steps = self.remove_duplicate_steps({**response.model_dump()}) + finished_steps = [step for step in self.current_state.steps if step["completed"]] + self.next_state.steps = finished_steps + [ + { + "id": uuid4().hex, + "completed": False, + "source": source, + "iteration_index": len(self.current_state.iterations), + **step, + } + for step in unique_steps["steps"] + ] + log.debug(f"Next steps: {self.next_state.unfinished_steps}") + + def remove_duplicate_steps(self, data): + unique_steps = [] + + # Process steps attribute + for step in data["steps"]: + if isinstance(step, SaveFileStep) and any( + s["type"] == "save_file" and s["save_file"]["path"] == step["save_file"]["path"] for s in unique_steps + ): + continue + unique_steps.append(step) + + # Update steps attribute + data["steps"] = unique_steps + + # Use the serializable_steps for JSON dumping + data["original_response"] = json.dumps(unique_steps, indent=2) + + return data + + async def ask_to_execute_task(self) -> bool: + """ + Asks the user to approve, skip or edit the current task. + + If task is edited, the method returns False so that the changes are saved. The + Orchestrator will rerun the agent on the next iteration. + + :return: True if the task should be executed as is, False if the task is skipped or edited + """ + buttons = {"yes": "Yes", "edit": "Edit Task"} + if len(self.current_state.tasks) > 1: + buttons["skip"] = "Skip Task" + + description = self.current_state.current_task["description"] + task_index = self.current_state.tasks.index(self.current_state.current_task) + 1 + await self.ui.send_project_stage( + { + "stage": ProjectStage.STARTING_TASK, + "task_index": task_index, + } + ) + await self.send_message(f"Starting task #{task_index} with the description:\n\n" + description) + if self.current_state.run_command: + await self.ui.send_run_command(self.current_state.run_command) + user_response = await self.ask_question( + "Do you want to execute the above task?", + buttons=buttons, + default="yes", + buttons_only=True, + hint=description, + ) + if user_response.button == "yes": + # Execute the task as is + return True + + if user_response.cancelled or user_response.button == "skip": + log.info(f"Skipping task: {description}") + self.next_state.current_task["instructions"] = "(skipped on user request)" + self.next_state.set_current_task_status(TaskStatus.SKIPPED) + await self.send_message("Skipping task...") + # We're done here, and will pick up the next task (if any) on the next run + return False + + user_response = await self.ask_question( + "Edit the task description:", + buttons={ + # FIXME: must be lowercase becase VSCode doesn't recognize it otherwise. Needs a fix in the extension + "continue": "continue", + "cancel": "Cancel", + }, + default="continue", + initial_text=description, + ) + if user_response.button == "cancel" or user_response.cancelled: + # User hasn't edited the task, so we can execute it immediately as is + return await self.ask_to_execute_task() + + self.next_state.current_task["description"] = user_response.text + self.next_state.current_task["run_always"] = True + self.next_state.relevant_files = None + log.info(f"Task description updated to: {user_response.text}") + # Orchestrator will rerun us with the new task description + return False + + async def update_knowledge_base(self): + """ + Update the knowledge base with the current task and steps. + """ + await self.state_manager.update_utility_functions(self.current_state.current_step) + self.next_state.complete_step("utility_function") + return AgentResponse.done(self) diff --git a/core/agents/error_handler.py b/core/agents/error_handler.py new file mode 100644 index 00000000..b48968f4 --- /dev/null +++ b/core/agents/error_handler.py @@ -0,0 +1,121 @@ +from uuid import uuid4 + +from core.agents.base import BaseAgent +from core.agents.convo import AgentConvo +from core.agents.response import AgentResponse +from core.db.models.project_state import IterationStatus +from core.log import get_logger + +log = get_logger(__name__) + + +class ErrorHandler(BaseAgent): + """ + Error handler agent. + + Error handler is responsible for handling errors returned by other agents. If it's possible + to recover from the error, it should do it (which may include updating the "next" state) and + return DONE. Otherwise it should return EXIT to tell Orchestrator to quit the application. + """ + + agent_type = "error-handler" + display_name = "Error Handler" + + async def run(self) -> AgentResponse: + from core.agents.executor import Executor + from core.agents.spec_writer import SpecWriter + + error = self.prev_response + if error is None: + log.warning("ErrorHandler called without a previous error", stack_info=True) + return AgentResponse.done(self) + + log.error( + f"Agent {error.agent.display_name} returned error response: {error.type}", + extra={"data": error.data}, + ) + + if isinstance(error.agent, SpecWriter): + # If SpecWriter wasn't able to get the project description, there's nothing for + # us to do. + return AgentResponse.exit(self) + + if isinstance(error.agent, Executor): + return await self.handle_command_error( + error.data.get("message", "Unknown error"), error.data.get("details", {}) + ) + + log.error( + f"Unhandled error response from agent {error.agent.display_name}", + extra={"data": error.data}, + ) + return AgentResponse.exit(self) + + async def handle_command_error(self, message: str, details: dict) -> AgentResponse: + """ + Handle an error returned by Executor agent. + + Error message must be the analyis of the command execution, and the details must contain: + * cmd - command that was executed + * timeout - timeout for the command if any (or None if no timeout was used) + * status_code - exit code for the command (or None if the command timed out) + * stdout - standard output of the command + * stderr - standard error of the command + + :return: AgentResponse + """ + cmd = details.get("cmd") + timeout = details.get("timeout") + status_code = details.get("status_code") + stdout = details.get("stdout", "") + stderr = details.get("stderr", "") + + if not message: + raise ValueError("No error message provided in command error response") + if not cmd: + raise ValueError("No command provided in command error response details") + + confirm = await self.ask_question( + "Can I debug why this command failed?", + buttons={"yes": "Yes", "no": "No"}, + default="yes", + buttons_only=True, + ) + if confirm.cancelled or confirm.button == "no": + log.info("Skipping command error debug (requested by user)") + return AgentResponse.done(self) + + llm = self.get_llm(stream_output=True) + convo = AgentConvo(self).template( + "debug", + task_steps=self.current_state.steps, + current_task=self.current_state.current_task, + # FIXME: can this break? + step_index=self.current_state.steps.index(self.current_state.current_step), + cmd=cmd, + timeout=timeout, + stdout=stdout, + stderr=stderr, + status_code=status_code, + # fixme: everything above copypasted from Executor + analysis=message, + ) + llm_response: str = await llm(convo) + + # TODO: duplicate from Troubleshooter, maybe extract to a ProjectState method? + self.next_state.iterations = self.current_state.iterations + [ + { + "id": uuid4().hex, + "user_feedback": f"Error running command: {cmd}", + "user_feedback_qa": None, + "description": llm_response, + "alternative_solutions": [], + "attempts": 1, + "status": IterationStatus.IMPLEMENT_SOLUTION, + "bug_hunting_cycles": [], + } + ] + # TODO: maybe have ProjectState.finished_steps as well? would make the debug/ran_command prompts nicer too + self.next_state.steps = [s for s in self.current_state.steps if s.get("completed") is True] + # No need to call complete_step() here as we've just removed the steps so that Developer can break down the iteration + return AgentResponse.done(self) diff --git a/core/agents/executor.py b/core/agents/executor.py new file mode 100644 index 00000000..865a74bd --- /dev/null +++ b/core/agents/executor.py @@ -0,0 +1,180 @@ +from datetime import datetime, timezone +from typing import Optional + +from pydantic import BaseModel, Field + +from core.agents.base import BaseAgent +from core.agents.convo import AgentConvo +from core.agents.response import AgentResponse +from core.llm.parser import JSONParser +from core.log import get_logger +from core.proc.exec_log import ExecLog +from core.proc.process_manager import ProcessManager +from core.state.state_manager import StateManager +from core.ui.base import AgentSource, UIBase, UISource + +log = get_logger(__name__) + +CMD_OUTPUT_SOURCE_NAME = "Command output" +CMD_OUTPUT_SOURCE_TYPE = "cli-output" + + +class CommandResult(BaseModel): + """ + Analysis of the command run and decision on the next steps. + """ + + analysis: str = Field( + description="Analysis of the command output (stdout, stderr) and exit code, in context of the current task" + ) + success: bool = Field( + description="True if the command should be treated as successful and the task should continue, false if the command unexpectedly failed and we should debug the issue" + ) + + +class Executor(BaseAgent): + agent_type = "executor" + display_name = "Executor" + + def __init__( + self, + state_manager: StateManager, + ui: UIBase, + ): + """ + Create a new Executor agent + """ + self.ui_source = AgentSource(self.display_name, self.agent_type) + self.cmd_ui_source = UISource(CMD_OUTPUT_SOURCE_NAME, CMD_OUTPUT_SOURCE_TYPE) + + self.ui = ui + self.state_manager = state_manager + self.process_manager = ProcessManager( + root_dir=state_manager.get_full_project_root(), + output_handler=self.output_handler, + exit_handler=self.exit_handler, + ) + + def for_step(self, step): + # FIXME: not needed, refactor to use self.current_state.current_step + # in general, passing current step is not needed + self.step = step + return self + + async def output_handler(self, out, err): + await self.ui.send_stream_chunk(out, source=self.cmd_ui_source) + await self.ui.send_stream_chunk(err, source=self.cmd_ui_source) + + async def exit_handler(self, process): + pass + + async def run(self) -> AgentResponse: + if not self.step: + raise ValueError("No current step set (probably an Orchestrator bug)") + + options = self.step["command"] + cmd = options["command"] + cmd_name = cmd[:30] + "..." if len(cmd) > 33 else cmd + timeout = options.get("timeout") + + if timeout: + q = f"Can I run command: {cmd} with {timeout}s timeout?" + else: + q = f"Can I run command: {cmd}?" + + confirm = await self.ask_question( + q, + buttons={"yes": "Yes", "no": "No"}, + default="yes", + buttons_only=False, + initial_text=cmd, + extra_info="remove_button_yes", + ) + if confirm.button == "no": + log.info(f"Skipping command execution of `{cmd}` (requested by user)") + await self.send_message(f"Skipping command {cmd}") + self.complete() + self.next_state.action = f'Skip "{cmd_name}"' + return AgentResponse.done(self) + + if confirm.button != "yes": + cmd = confirm.text + + started_at = datetime.now(timezone.utc) + + log.info(f"Running command `{cmd}` with timeout {timeout}s") + status_code, stdout, stderr = await self.process_manager.run_command(cmd, timeout=timeout) + + llm_response = await self.check_command_output(cmd, timeout, stdout, stderr, status_code) + + duration = (datetime.now(timezone.utc) - started_at).total_seconds() + + self.complete() + self.next_state.action = f'Run "{cmd_name}"' + + exec_log = ExecLog( + started_at=started_at, + duration=duration, + cmd=cmd, + cwd=".", + env={}, + timeout=timeout, + status_code=status_code, + stdout=stdout, + stderr=stderr, + analysis=llm_response.analysis, + success=llm_response.success, + ) + await self.state_manager.log_command_run(exec_log) + + # FIXME: ErrorHandler isn't debugged with BugHunter - we should move all commands to run before testing and debug them with BugHunter + if True or llm_response.success: + return AgentResponse.done(self) + + return AgentResponse.error( + self, + llm_response.analysis, + { + "cmd": cmd, + "timeout": timeout, + "stdout": stdout, + "stderr": stderr, + "status_code": status_code, + }, + ) + + async def check_command_output( + self, cmd: str, timeout: Optional[int], stdout: str, stderr: str, status_code: int + ) -> CommandResult: + llm = self.get_llm() + convo = ( + AgentConvo(self) + .template( + "ran_command", + task_steps=self.current_state.steps, + current_task=self.current_state.current_task, + # FIXME: can step ever happen *not* to be in current steps? + step_index=self.current_state.steps.index(self.step), + cmd=cmd, + timeout=timeout, + stdout=stdout, + stderr=stderr, + status_code=status_code, + ) + .require_schema(CommandResult) + ) + return await llm(convo, parser=JSONParser(spec=CommandResult), temperature=0) + + def complete(self): + """ + Mark the step as complete. + + Note that this marks the step complete in the next state. If there's an error, + the state won't get committed and the error handler will have access to the + current state, where this step is still unfinished. + + This is intentional, so that the error handler can decide what to do with the + information we give it. + """ + self.step = None + self.next_state.complete_step("command") diff --git a/core/agents/external_docs.py b/core/agents/external_docs.py new file mode 100644 index 00000000..a5f6c999 --- /dev/null +++ b/core/agents/external_docs.py @@ -0,0 +1,173 @@ +import asyncio +from urllib.parse import urljoin + +import httpx +from pydantic import BaseModel + +from core.agents.base import BaseAgent +from core.agents.convo import AgentConvo +from core.agents.response import AgentResponse +from core.config import EXTERNAL_DOCUMENTATION_API +from core.llm.parser import JSONParser +from core.log import get_logger +from core.telemetry import telemetry + +log = get_logger(__name__) + + +class DocQueries(BaseModel): + queries: list[str] + + +class SelectedDocsets(BaseModel): + docsets: list[str] + + +class ExternalDocumentation(BaseAgent): + """Agent in charge of collecting and storing additional documentation. + + Docs are per task and are stores in the `docs` variable in the project state. + This agent ensures documentation is collected only once per task. + + Agent does 2 LLM interactions: + 1. Ask the LLM to select useful documentation from a predefined list. + 2. Ask the LLM to come up with a query to use to fetch the actual documentation snippets. + + Agent does 2 calls to our documentation API: + 1. Fetch all the available docsets. `docset` is a collection of documentation snippets + for a single topic, eg. VueJS API Reference docs. + 2. Fetch the documentation snippets for given queries. + + """ + + agent_type = "external-docs" + display_name = "Documentation" + + async def run(self) -> AgentResponse: + await self._store_docs([], []) + return AgentResponse.done(self) + + if self.current_state.specification.example_project: + log.debug("Example project detected, no documentation selected.") + available_docsets = [] + else: + available_docsets = await self._get_available_docsets() + + selected_docsets = await self._select_docsets(available_docsets) + await telemetry.trace_code_event("docsets_used", selected_docsets) + + if not selected_docsets: + log.info("No documentation selected for this task.") + await self._store_docs([], available_docsets) + return AgentResponse.done(self) + + log.info(f"Selected {len(selected_docsets)} docsets for this task.") + queries = await self._create_queries(selected_docsets) + doc_snippets = await self._fetch_snippets(queries) + await telemetry.trace_code_event("doc_snippets", {"num_stored": len(doc_snippets)}) + + await self._store_docs(doc_snippets, available_docsets) + return AgentResponse.done(self) + + async def _get_available_docsets(self) -> list[tuple]: + url = urljoin(EXTERNAL_DOCUMENTATION_API, "docsets") + client = httpx.Client(transport=httpx.HTTPTransport(retries=3)) + try: + resp = client.get(url) + except httpx.HTTPError: + # In case of any errors, we'll proceed without the documentation + log.warning("Failed to fetch available docsets due to an error.", exc_info=True) + return [] + + log.debug(f"Fetched {len(resp.json())} docsets.") + return resp.json() + + async def _select_docsets(self, available_docsets: list[tuple]) -> dict[str, str]: + """From a list of available docsets, select the relevant ones.""" + + if not available_docsets: + return {} + + llm = self.get_llm(stream_output=True) + convo = ( + AgentConvo(self) + .template( + "select_docset", + current_task=self.current_state.current_task, + available_docsets=available_docsets, + ) + .require_schema(SelectedDocsets) + ) + await self.send_message("Determining if external documentation is needed for the next task...") + llm_response: SelectedDocsets = await llm(convo, parser=JSONParser(spec=SelectedDocsets)) + available_docsets = dict(available_docsets) + return {k: available_docsets[k] for k in llm_response.docsets if k in available_docsets} + + async def _create_queries(self, docsets: dict[str, str]) -> dict[str, list[str]]: + """Return queries we have to make to the docs API. + + Key is the docset_key and value is the list of queries for that docset. + + """ + queries = {} + await self.send_message("Getting relevant documentation for the following topics:") + for k, short_desc in docsets.items(): + llm = self.get_llm(stream_output=True) + convo = ( + AgentConvo(self) + .template( + "create_docs_queries", + short_description=short_desc, + current_task=self.current_state.current_task, + ) + .require_schema(DocQueries) + ) + llm_response: DocQueries = await llm(convo, parser=JSONParser(spec=DocQueries)) + if llm_response.queries: + queries[k] = llm_response.queries + + return queries + + async def _fetch_snippets(self, queries: dict[str, list[str]]) -> list[tuple]: + """Query the docs API and fetch the documentation snippets. + + Returns a list of tuples: (docset_key, snippets). + + """ + url = urljoin(EXTERNAL_DOCUMENTATION_API, "query") + snippets: list[tuple] = [] + async with httpx.AsyncClient(transport=httpx.AsyncHTTPTransport(retries=3)) as client: + reqs = [] + ordered_keys = [] + for docset_key, qs in queries.items(): + reqs.append(client.get(url, params={"q": qs, "doc_key": docset_key, "num_results": 3})) + ordered_keys.append(docset_key) + + try: + results = await asyncio.gather(*reqs) + except httpx.HTTPError: + log.warning("Failed to fetch documentation snippets", exc_info=True) + + for k, res in zip(ordered_keys, results): + json_snippets = res.json() + log.debug(f"Fetched {len(json_snippets)} snippets from {k}") + if len(json_snippets): + snippets.append((k, res.json())) + return snippets + + async def _store_docs(self, snippets: list[tuple], available_docsets: list[tuple]): + """Store the snippets into current task data. + + Documentation snippets are stored as a list of dictionaries: + {"key": docset-key, "desc": documentation-description, "snippets": list-of-snippets} + + :param snippets: List of tuples: (docset_key, snippets) + :param available_docsets: List of available docsets from the API. + """ + + docsets_dict = dict(available_docsets) + docs = [] + for docset_key, snip in snippets: + docs.append({"key": docset_key, "desc": docsets_dict[docset_key], "snippets": snip}) + + self.next_state.docs = docs diff --git a/core/agents/frontend.py b/core/agents/frontend.py new file mode 100644 index 00000000..3cd96491 --- /dev/null +++ b/core/agents/frontend.py @@ -0,0 +1,297 @@ +from uuid import uuid4 + +from core.agents.base import BaseAgent +from core.agents.convo import AgentConvo +from core.agents.mixins import FileDiffMixin +from core.agents.response import AgentResponse +from core.config import FRONTEND_AGENT_NAME +from core.llm.parser import DescriptiveCodeBlockParser +from core.log import get_logger +from core.telemetry import telemetry +from core.templates.registry import PROJECT_TEMPLATES +from core.ui.base import ProjectStage + +log = get_logger(__name__) + + +class Frontend(FileDiffMixin, BaseAgent): + agent_type = "frontend" + display_name = "Frontend" + + async def run(self) -> AgentResponse: + if not self.current_state.epics: + finished = await self.init_frontend() + elif not self.current_state.epics[0]["messages"]: + finished = await self.start_frontend() + elif not self.next_state.epics[-1].get("fe_iteration_done"): + finished = await self.continue_frontend() + else: + await self.set_app_details() + finished = await self.iterate_frontend() + + return await self.end_frontend_iteration(finished) + + async def init_frontend(self) -> bool: + """ + Builds frontend of the app. + + :return: AgentResponse.done(self) + """ + await self.ui.send_project_stage({"stage": ProjectStage.PROJECT_DESCRIPTION}) + + description = await self.ask_question( + "Please describe the app you want to build.", + allow_empty=False, + full_screen=True, + ) + description = description.text.strip() + + auth_needed = await self.ask_question( + "Do you need authentication in your app (login, register, etc.)?", + buttons={ + "yes": "Yes", + "no": "No", + }, + buttons_only=True, + default="no", + ) + options = { + "auth": auth_needed.button == "yes", + } + self.next_state.knowledge_base["user_options"] = options + self.state_manager.user_options = options + + await self.send_message("Setting up the project...") + + self.next_state.epics = [ + { + "id": uuid4().hex, + "name": "Build frontend", + "source": "frontend", + "description": description, + "messages": [], + "summary": None, + "completed": False, + } + ] + + await self.apply_template(options) + + return False + + async def start_frontend(self): + """ + Starts the frontend of the app. + """ + await self.send_message("Building the frontend... This may take a couple of minutes") + description = self.current_state.epics[0]["description"] + + llm = self.get_llm(FRONTEND_AGENT_NAME) + convo = AgentConvo(self).template( + "build_frontend", + description=description, + user_feedback=None, + ) + response = await llm(convo, parser=DescriptiveCodeBlockParser()) + response_blocks = response.blocks + convo.assistant(response.original_response) + + await self.process_response(response_blocks) + + self.next_state.epics[-1]["messages"] = convo.messages + self.next_state.epics[-1]["fe_iteration_done"] = ( + "done" in response.original_response[-20:].lower().strip() or len(convo.messages) > 11 + ) + self.next_state.flag_epics_as_modified() + + return False + + async def continue_frontend(self): + """ + Continues building the frontend of the app after the initial user input. + """ + await self.ui.send_project_stage({"stage": ProjectStage.CONTINUE_FRONTEND}) + await self.send_message("Continuing to build UI... This may take a couple of minutes") + + llm = self.get_llm(FRONTEND_AGENT_NAME) + convo = AgentConvo(self) + convo.messages = self.current_state.epics[0]["messages"] + convo.user( + "Ok, now think carefully about your previous response. If the response ends by mentioning something about continuing with the implementation, continue but don't implement any files that have already been implemented. If your last response doesn't end by mentioning continuing, respond only with `DONE` and with nothing else." + ) + response = await llm(convo, parser=DescriptiveCodeBlockParser()) + response_blocks = response.blocks + convo.assistant(response.original_response) + + await self.process_response(response_blocks) + + self.next_state.epics[-1]["messages"] = convo.messages + self.next_state.epics[-1]["fe_iteration_done"] = ( + "done" in response.original_response[-20:].lower().strip() or len(convo.messages) > 15 + ) + self.next_state.flag_epics_as_modified() + + return False + + async def iterate_frontend(self) -> bool: + """ + Iterates over the frontend. + + :return: True if the frontend is fully built, False otherwise. + """ + + # update the pages in the knowledge base + await self.state_manager.update_implemented_pages_and_apis() + + await self.ui.send_project_stage({"stage": ProjectStage.ITERATE_FRONTEND}) + + answer = await self.ask_question( + "Do you want to change anything or report a bug? Keep in mind that currently ONLY frontend is implemented.", + buttons={ + "yes": "I'm done building the UI", + }, + default="yes", + extra_info="restart_app/collect_logs", + placeholder='For example, "I don\'t see anything when I open http://localhost:5173/" or "Nothing happens when I click on the NEW PROJECT button"', + ) + + if answer.button == "yes": + answer = await self.ask_question( + "Are you sure you're done building the UI and want to start building the backend functionality now?", + buttons={ + "yes": "Yes, let's build the backend", + "no": "No, continue working on the UI", + }, + buttons_only=True, + default="yes", + ) + + if answer.button == "yes": + return True + else: + return False + + await self.send_message("Implementing the changes you suggested...") + + llm = self.get_llm(FRONTEND_AGENT_NAME, stream_output=True) + convo = AgentConvo(self).template( + "build_frontend", + description=self.current_state.epics[0]["description"], + user_feedback=answer.text, + ) + response = await llm(convo, parser=DescriptiveCodeBlockParser()) + + await self.process_response(response.blocks) + + return False + + async def end_frontend_iteration(self, finished: bool) -> AgentResponse: + """ + Ends the frontend iteration. + + :param finished: Whether the frontend is fully built. + :return: AgentResponse.done(self) + """ + if finished: + # TODO Add question if user app is fully finished + self.next_state.complete_epic() + await telemetry.trace_code_event( + "frontend-finished", + { + "description": self.current_state.epics[0]["description"], + "messages": self.current_state.epics[0]["messages"], + }, + ) + + inputs = [] + for file in self.current_state.files: + if not file.content: + continue + input_required = self.state_manager.get_input_required(file.content.content, file.path) + if input_required: + inputs += [{"file": file.path, "line": line} for line in input_required] + + if inputs: + return AgentResponse.input_required(self, inputs) + + return AgentResponse.done(self) + + async def process_response(self, response_blocks: list) -> AgentResponse: + """ + Processes the response blocks from the LLM. + + :param response_blocks: The response blocks from the LLM. + :return: AgentResponse.done(self) + """ + for block in response_blocks: + description = block.description.strip() + content = block.content.strip() + + # Split description into lines and check the last line for file path + description_lines = description.split("\n") + last_line = description_lines[-1].strip() + + if "file:" in last_line: + # Extract file path from the last line - get everything after "file:" + file_path = last_line[last_line.index("file:") + 5 :].strip() + file_path = file_path.strip("\"'`") + new_content = content + old_content = self.current_state.get_file_content_by_path(file_path) + n_new_lines, n_del_lines = self.get_line_changes(old_content, new_content) + await self.ui.send_file_status(file_path, "done", source=self.ui_source) + await self.ui.generate_diff( + file_path, old_content, new_content, n_new_lines, n_del_lines, source=self.ui_source + ) + await self.state_manager.save_file(file_path, new_content) + + elif "command:" in last_line: + # Split multiple commands and execute them sequentially + commands = content.strip().split("\n") + for command in commands: + command = command.strip() + if command: + # Add "cd client" prefix if not already present + if not command.startswith("cd "): + command = f"cd client && {command}" + await self.send_message(f"Running command: `{command}`...") + await self.process_manager.run_command(command) + else: + log.info(f"Unknown block description: {description}") + + return AgentResponse.done(self) + + async def apply_template(self, options: dict = {}): + """ + Applies a template to the frontend. + """ + template_name = "vite_react" + template_class = PROJECT_TEMPLATES.get(template_name) + if not template_class: + log.error(f"Project template not found: {template_name}") + return + + template = template_class( + options, + self.state_manager, + self.process_manager, + ) + + log.info(f"Applying project template: {template.name}") + summary = await template.apply() + + self.next_state.relevant_files = template.relevant_files + self.next_state.modified_files = {} + self.next_state.specification.template_summary = summary + + async def set_app_details(self): + """ + Sets the app details. + """ + command = "npm run start" + app_link = "http://localhost:5173" + + self.next_state.run_command = command + # todo store app link and send whenever we are sending run_command + # self.next_state.app_link = app_link + await self.ui.send_run_command(command) + await self.ui.send_app_link(app_link) diff --git a/core/agents/git.py b/core/agents/git.py new file mode 100644 index 00000000..3cb2cf54 --- /dev/null +++ b/core/agents/git.py @@ -0,0 +1,165 @@ +import os + +from core.agents.convo import AgentConvo +from core.config.magic_words import GITIGNORE_CONTENT +from core.ui.base import pythagora_source + + +class GitMixin: + """ + Mixin class for git commands + """ + + async def check_git_installed(self) -> bool: + """Check if git is installed on the system.""" + status_code, _, _ = await self.process_manager.run_command("git --version", show_output=False) + git_available = status_code == 0 + self.state_manager.git_available = git_available + return git_available + + async def is_git_initialized(self) -> bool: + """Check if git is initialized in the workspace.""" + workspace_path = self.state_manager.get_full_project_root() + + status_code, _, _ = await self.process_manager.run_command( + "git rev-parse --git-dir --is-inside-git-dir", + cwd=workspace_path, + show_output=False, + ) + # Will return status code 0 only if .git exists in the current directory + git_used = status_code == 0 and os.path.exists(os.path.join(workspace_path, ".git")) + self.state_manager.git_used = git_used + return git_used + + async def init_git_if_needed(self) -> bool: + """ + Initialize git repository if it hasn't been initialized yet. + Returns True if initialization was needed and successful. + """ + + workspace_path = self.state_manager.get_full_project_root() + if await self.is_git_initialized(): + return False + + answer = await self.ui.ask_question( + "Git is not initialized for this project. Do you want to initialize it now?", + buttons={"yes": "Yes", "no": "No"}, + default="yes", + buttons_only=True, + source=pythagora_source, + ) + + if answer.button == "no": + return False + else: + status_code, _, stderr = await self.process_manager.run_command("git init", cwd=workspace_path) + if status_code != 0: + raise RuntimeError(f"Failed to initialize git repository: {stderr}") + + gitignore_path = os.path.join(workspace_path, ".gitignore") + try: + with open(gitignore_path, "w") as f: + f.write(GITIGNORE_CONTENT) + except Exception as e: + raise RuntimeError(f"Failed to create .gitignore file: {str(e)}") + + # First check if there are any changes to commit + status_code, stdout, stderr = await self.process_manager.run_command( + "git status --porcelain", + cwd=workspace_path, + ) + + if status_code == 0 and stdout.strip(): # If there are changes (stdout is not empty) + # Stage all files + status_code, _, stderr = await self.process_manager.run_command( + "git add .", + cwd=workspace_path, + ) + if status_code != 0: + raise RuntimeError(f"Failed to stage files: {stderr}") + + # Create initial commit + status_code, _, stderr = await self.process_manager.run_command( + 'git commit -m "initial commit"', cwd=workspace_path + ) + if status_code != 0: + raise RuntimeError(f"Failed to create initial commit: {stderr}") + + self.state_manager.git_used = True + return True + + async def git_commit(self) -> None: + """ + Create a git commit with the specified message. + Raises RuntimeError if the commit fails. + """ + workspace_path = self.state_manager.get_full_project_root() + + # Check if there are any changes to commit + status_code, git_status, stderr = await self.process_manager.run_command( + "git status --porcelain", + cwd=workspace_path, + show_output=False, + ) + if status_code != 0: + raise RuntimeError(f"Failed to get git status: {stderr}") + + if not git_status.strip(): + return + + answer = await self.ui.ask_question( + "Do you want to create new git commit?", + buttons={"yes": "Yes", "no": "No"}, + default="yes", + buttons_only=True, + source=pythagora_source, + ) + + if answer.button == "no": + return + + # Stage all changes + status_code, _, stderr = await self.process_manager.run_command("git add .", cwd=workspace_path) + if status_code != 0: + raise RuntimeError(f"Failed to stage changes: {stderr}") + + # Get git diff + status_code, git_diff, stderr = await self.process_manager.run_command( + "git diff --cached || git diff", + cwd=workspace_path, + show_output=False, + ) + if status_code != 0: + raise RuntimeError(f"Failed to create initial commit: {stderr}") + + llm = self.get_llm() + convo = AgentConvo(self).template( + "commit", + git_diff=git_diff, + ) + commit_message: str = await llm(convo) + + answer = await self.ui.ask_question( + f"Do you accept this 'git commit' message? Here is suggested message: '{commit_message}'", + buttons={"yes": "Yes", "edit": "Edit", "no": "No, I don't want to commit changes."}, + default="yes", + buttons_only=True, + source=pythagora_source, + ) + + if answer.button == "no": + return + elif answer.button == "edit": + user_message = await self.ui.ask_question( + "Please enter the commit message", + source=pythagora_source, + initial_text=commit_message, + ) + commit_message = user_message.text + + # Create commit + status_code, _, stderr = await self.process_manager.run_command( + f'git commit -m "{commit_message}"', cwd=workspace_path + ) + if status_code != 0: + raise RuntimeError(f"Failed to create commit: {stderr}") diff --git a/core/agents/human_input.py b/core/agents/human_input.py new file mode 100644 index 00000000..3c0ac2f4 --- /dev/null +++ b/core/agents/human_input.py @@ -0,0 +1,46 @@ +from core.agents.base import BaseAgent +from core.agents.response import AgentResponse, ResponseType + + +class HumanInput(BaseAgent): + agent_type = "human-input" + display_name = "Human Input" + + async def run(self) -> AgentResponse: + if self.prev_response and self.prev_response.type == ResponseType.INPUT_REQUIRED: + return await self.input_required(self.prev_response.data.get("files", [])) + + return await self.human_intervention(self.step) + + async def human_intervention(self, step) -> AgentResponse: + description = step["human_intervention_description"] + + await self.ask_question( + f"I need human intervention: {description}", + buttons={"continue": "Continue"}, + default="continue", + buttons_only=True, + ) + self.next_state.complete_step("human_intervention") + return AgentResponse.done(self) + + async def input_required(self, files: list[dict]) -> AgentResponse: + for item in files: + file = item["file"] + line = item["line"] + + # FIXME: this is an ugly hack, we shouldn't need to know how to get to VFS and + # anyways the full path is only available for local vfs, so this is doubly wrong; + # instead, we should just send the relative path to the extension and it should + # figure out where its local files are and how to open it. + full_path = self.state_manager.file_system.get_full_path(file) + + await self.send_message(f"Input required on {file}:{line}") + await self.ui.open_editor(full_path, line) + await self.ask_question( + f"Please open {file} and modify line {line} according to the instructions.", + buttons={"continue": "Continue"}, + default="continue", + buttons_only=True, + ) + return AgentResponse.done(self) diff --git a/core/agents/importer.py b/core/agents/importer.py new file mode 100644 index 00000000..3704c930 --- /dev/null +++ b/core/agents/importer.py @@ -0,0 +1,97 @@ +from uuid import uuid4 + +from core.agents.base import BaseAgent +from core.agents.convo import AgentConvo +from core.agents.response import AgentResponse, ResponseType +from core.db.models import Complexity +from core.llm.parser import JSONParser +from core.log import get_logger +from core.telemetry import telemetry +from core.templates.example_project import EXAMPLE_PROJECT_DESCRIPTION + +log = get_logger(__name__) + +MAX_PROJECT_LINES = 10000 + + +class Importer(BaseAgent): + agent_type = "importer" + display_name = "Project Analyist" + + async def run(self) -> AgentResponse: + if self.prev_response and self.prev_response.type == ResponseType.IMPORT_PROJECT: + # Called by SpecWriter to start the import process + await self.start_import_process() + return AgentResponse.describe_files(self) + + await self.analyze_project() + return AgentResponse.done(self) + + async def start_import_process(self): + # TODO: Send a signal to the UI to copy the project files to workspace + project_root = self.state_manager.get_full_project_root() + await self.ui.import_project(project_root) + await self.send_message( + f"This is experimental feature and is currently limited to projects with size up to {MAX_PROJECT_LINES} lines of code." + ) + + await self.ask_question( + f"Please copy your project files to {project_root} and press Continue", + allow_empty=False, + buttons={ + "continue": "Continue", + }, + buttons_only=True, + default="continue", + ) + + imported_files, _ = await self.state_manager.import_files() + imported_lines = sum(len(f.content.content.splitlines()) for f in imported_files) + if imported_lines > MAX_PROJECT_LINES: + await self.send_message( + "WARNING: Your project ({imported_lines} LOC) is larger than supported and may cause issues in Pythagora." + ) + await self.state_manager.commit() + + async def analyze_project(self): + llm = self.get_llm(stream_output=True) + + self.send_message("Inspecting most important project files ...") + + convo = AgentConvo(self).template("get_entrypoints") + llm_response = await llm(convo, parser=JSONParser()) + relevant_files = [f for f in self.current_state.files if f.path in llm_response] + + self.send_message("Analyzing project ...") + + convo = AgentConvo(self).template( + "analyze_project", relevant_files=relevant_files, example_spec=EXAMPLE_PROJECT_DESCRIPTION + ) + llm_response = await llm(convo) + + spec = self.current_state.specification.clone() + spec.description = llm_response + self.next_state.specification = spec + self.next_state.epics = [ + { + "id": uuid4().hex, + "name": "Import project", + "description": "Import an existing project into Pythagora", + "tasks": [], + "completed": True, + "test_instructions": None, + "source": "app", + "summary": None, + "complexity": Complexity.HARD if len(self.current_state.files) > 5 else Complexity.SIMPLE, + } + ] + + n_lines = sum(len(f.content.content.splitlines()) for f in self.current_state.files) + await telemetry.trace_code_event( + "existing-project", + { + "num_files": len(self.current_state.files), + "num_lines": n_lines, + "description": llm_response, + }, + ) diff --git a/core/agents/legacy_handler.py b/core/agents/legacy_handler.py new file mode 100644 index 00000000..d046f0e2 --- /dev/null +++ b/core/agents/legacy_handler.py @@ -0,0 +1,14 @@ +from core.agents.base import BaseAgent +from core.agents.response import AgentResponse + + +class LegacyHandler(BaseAgent): + agent_type = "legacy-handler" + display_name = "Legacy Handler" + + async def run(self) -> AgentResponse: + if self.data["type"] == "review_task": + self.next_state.complete_step("review_task") + return AgentResponse.done(self) + + raise ValueError(f"Unknown reason for calling Legacy Handler with data: {self.data}") diff --git a/core/agents/mixins.py b/core/agents/mixins.py new file mode 100644 index 00000000..67318787 --- /dev/null +++ b/core/agents/mixins.py @@ -0,0 +1,225 @@ +import json +from difflib import unified_diff +from typing import List, Optional, Union + +from pydantic import BaseModel, Field + +from core.agents.convo import AgentConvo +from core.agents.response import AgentResponse +from core.config import GET_RELEVANT_FILES_AGENT_NAME, TASK_BREAKDOWN_AGENT_NAME, TROUBLESHOOTER_BUG_REPORT +from core.llm.parser import JSONParser +from core.log import get_logger +from core.ui.base import ProjectStage + +log = get_logger(__name__) + + +class ReadFilesAction(BaseModel): + read_files: Optional[List[str]] = Field( + description="List of files you want to read. All listed files must be in the project." + ) + + +class AddFilesAction(BaseModel): + add_files: Optional[List[str]] = Field( + description="List of files you want to add to the list of relevant files. All listed files must be in the project. You must read files before adding them." + ) + + +class RemoveFilesAction(BaseModel): + remove_files: Optional[List[str]] = Field( + description="List of files you want to remove from the list of relevant files. All listed files must be in the relevant files list." + ) + + +class DoneBooleanAction(BaseModel): + done: Optional[bool] = Field(description="Boolean flag to indicate that you are done creating breakdown.") + + +class RelevantFiles(BaseModel): + action: Union[ReadFilesAction, AddFilesAction, RemoveFilesAction, DoneBooleanAction] + + +class Test(BaseModel): + title: str = Field(description="Very short title of the test.") + action: str = Field(description="More detailed description of what actions have to be taken to test the app.") + result: str = Field(description="Expected result that verifies successful test.") + + +class TestSteps(BaseModel): + steps: List[Test] + + +class ChatWithBreakdownMixin: + """ + Provides a method to chat with the user and provide a breakdown of the conversation. + """ + + async def chat_with_breakdown(self, convo: AgentConvo, breakdown: str) -> AgentConvo: + """ + Chat with the user and provide a breakdown of the conversation. + + :param convo: The conversation object. + :param breakdown: The breakdown of the conversation. + :return: The breakdown. + """ + + llm = self.get_llm(TASK_BREAKDOWN_AGENT_NAME, stream_output=True) + while True: + await self.ui.send_project_stage( + { + "stage": ProjectStage.BREAKDOWN_CHAT, + "agent": self.agent_type, + } + ) + + chat = await self.ask_question( + "Are you happy with the breakdown? Now is a good time to ask questions or suggest changes.", + buttons={"yes": "Yes, looks good!"}, + default="yes", + verbose=False, + ) + if chat.button == "yes": + break + + if len(convo.messages) > 11: + convo.trim(3, 2) + + convo.user(chat.text) + breakdown: str = await llm(convo) + convo.assistant(breakdown) + + return breakdown + + +class IterationPromptMixin: + """ + Provides a method to find a solution to a problem based on user feedback. + + Used by ProblemSolver and Troubleshooter agents. + """ + + async def find_solution( + self, + user_feedback: str, + *, + user_feedback_qa: Optional[list[str]] = None, + next_solution_to_try: Optional[str] = None, + bug_hunting_cycles: Optional[dict] = None, + ) -> str: + """ + Generate a new solution for the problem the user reported. + + :param user_feedback: User feedback about the problem. + :param user_feedback_qa: Additional q/a about the problem provided by the user (optional). + :param next_solution_to_try: Hint from ProblemSolver on which solution to try (optional). + :param bug_hunting_cycles: Data about logs that need to be added to the code (optional). + :return: The generated solution to the problem. + """ + llm = self.get_llm(TROUBLESHOOTER_BUG_REPORT, stream_output=True) + convo = AgentConvo(self).template( + "iteration", + user_feedback=user_feedback, + user_feedback_qa=user_feedback_qa, + next_solution_to_try=next_solution_to_try, + bug_hunting_cycles=bug_hunting_cycles, + test_instructions=json.loads(self.current_state.current_task.get("test_instructions") or "[]"), + ) + llm_solution: str = await llm(convo) + + llm_solution = await self.chat_with_breakdown(convo, llm_solution) + + return llm_solution + + +class RelevantFilesMixin: + """ + Provides a method to get relevant files for the current task. + """ + + async def get_relevant_files( + self, user_feedback: Optional[str] = None, solution_description: Optional[str] = None + ) -> AgentResponse: + log.debug("Getting relevant files for the current task") + done = False + relevant_files = set() + llm = self.get_llm(GET_RELEVANT_FILES_AGENT_NAME) + convo = ( + AgentConvo(self) + .template( + "filter_files", + user_feedback=user_feedback, + solution_description=solution_description, + relevant_files=relevant_files, + ) + .require_schema(RelevantFiles) + ) + + while not done and len(convo.messages) < 23: + llm_response: RelevantFiles = await llm(convo, parser=JSONParser(RelevantFiles), temperature=0) + action = llm_response.action + if action is None: + convo.remove_last_x_messages(2) + continue + + # Check if there are files to add to the list + if getattr(action, "add_files", None): + # Add only the files from add_files that are not already in relevant_files + relevant_files.update(file for file in action.add_files if file not in relevant_files) + + # Check if there are files to remove from the list + if getattr(action, "remove_files", None): + # Remove files from relevant_files that are in remove_files + relevant_files.difference_update(action.remove_files) + + read_files = [ + file for file in self.current_state.files if file.path in (getattr(action, "read_files", []) or []) + ] + + convo.remove_last_x_messages(1) + convo.assistant(llm_response.original_response) + convo.template("filter_files_loop", read_files=read_files, relevant_files=relevant_files).require_schema( + RelevantFiles + ) + done = getattr(action, "done", False) + + existing_files = {file.path for file in self.current_state.files} + relevant_files = [path for path in relevant_files if path in existing_files] + self.current_state.relevant_files = relevant_files + self.next_state.relevant_files = relevant_files + + return AgentResponse.done(self) + + +class FileDiffMixin: + """ + Provides a method to generate a diff between two files. + """ + + def get_line_changes(self, old_content: str, new_content: str) -> tuple[int, int]: + """ + Get the number of added and deleted lines between two files. + + This uses Python difflib to produce a unified diff, then counts + the number of added and deleted lines. + + :param old_content: old file content + :param new_content: new file content + :return: a tuple (added_lines, deleted_lines) + """ + + from_lines = old_content.splitlines(keepends=True) + to_lines = new_content.splitlines(keepends=True) + + diff_gen = unified_diff(from_lines, to_lines) + + added_lines = 0 + deleted_lines = 0 + + for line in diff_gen: + if line.startswith("+") and not line.startswith("+++"): # Exclude the file headers + added_lines += 1 + elif line.startswith("-") and not line.startswith("---"): # Exclude the file headers + deleted_lines += 1 + + return added_lines, deleted_lines diff --git a/core/agents/orchestrator.py b/core/agents/orchestrator.py new file mode 100644 index 00000000..8a75e00d --- /dev/null +++ b/core/agents/orchestrator.py @@ -0,0 +1,409 @@ +import asyncio +import os +from typing import List, Optional, Union + +from core.agents.architect import Architect +from core.agents.base import BaseAgent +from core.agents.bug_hunter import BugHunter +from core.agents.code_monkey import CodeMonkey +from core.agents.developer import Developer +from core.agents.error_handler import ErrorHandler +from core.agents.executor import Executor +from core.agents.external_docs import ExternalDocumentation +from core.agents.frontend import Frontend +from core.agents.git import GitMixin +from core.agents.human_input import HumanInput +from core.agents.importer import Importer +from core.agents.legacy_handler import LegacyHandler +from core.agents.problem_solver import ProblemSolver +from core.agents.response import AgentResponse, ResponseType +from core.agents.spec_writer import SpecWriter +from core.agents.task_completer import TaskCompleter +from core.agents.tech_lead import TechLead +from core.agents.tech_writer import TechnicalWriter +from core.agents.troubleshooter import Troubleshooter +from core.db.models.project_state import IterationStatus, TaskStatus +from core.log import get_logger +from core.telemetry import telemetry + +log = get_logger(__name__) + + +class Orchestrator(BaseAgent, GitMixin): + """ + Main agent that controls the flow of the process. + + Based on the current state of the project, the orchestrator invokes + all other agents. It is also responsible for determining when each + step is done and the project state needs to be committed to the database. + """ + + agent_type = "orchestrator" + display_name = "Orchestrator" + + async def run(self) -> bool: + """ + Run the Orchestrator agent. + + :return: True if the Orchestrator exited successfully, False otherwise. + """ + response = None + + log.info(f"Starting {__name__}.Orchestrator") + + self.executor = Executor(self.state_manager, self.ui) + self.process_manager = self.executor.process_manager + # self.chat = Chat() TODO + + await self.init_ui() + await self.offline_changes_check() + + await self.install_dependencies() + + if self.args.use_git and await self.check_git_installed(): + await self.init_git_if_needed() + + # TODO: consider refactoring this into two loop; the outer with one iteration per comitted step, + # and the inner which runs the agents for the current step until they're done. This would simplify + # handle_done() and let us do other per-step processing (eg. describing files) in between agent runs. + while True: + await self.update_stats() + + agent = self.create_agent(response) + + # In case where agent is a list, run all agents in parallel. + # Only one agent type can be run in parallel at a time (for now). See handle_parallel_responses(). + if isinstance(agent, list): + tasks = [single_agent.run() for single_agent in agent] + log.debug( + f"Running agents {[a.__class__.__name__ for a in agent]} (step {self.current_state.step_index})" + ) + responses = await asyncio.gather(*tasks) + response = self.handle_parallel_responses(agent[0], responses) + + should_update_knowledge_base = any( + "src/pages/" in single_agent.step.get("save_file", {}).get("path", "") + or "src/api/" in single_agent.step.get("save_file", {}).get("path", "") + or len(single_agent.step.get("related_api_endpoints")) > 0 + for single_agent in agent + ) + + if should_update_knowledge_base: + files_with_implemented_apis = [ + { + "path": single_agent.step.get("save_file", {}).get("path", None), + "related_api_endpoints": single_agent.step.get("related_api_endpoints"), + "line": 0, # TODO implement getting the line number here + } + for single_agent in agent + if len(single_agent.step.get("related_api_endpoints")) > 0 + ] + await self.state_manager.update_apis(files_with_implemented_apis) + await self.state_manager.update_implemented_pages_and_apis() + + else: + log.debug(f"Running agent {agent.__class__.__name__} (step {self.current_state.step_index})") + response = await agent.run() + + if response.type == ResponseType.EXIT: + log.debug(f"Agent {agent.__class__.__name__} requested exit") + break + + if response.type == ResponseType.DONE: + response = await self.handle_done(agent, response) + continue + + # TODO: rollback changes to "next" so they aren't accidentally committed? + return True + + async def install_dependencies(self): + # First check if package.json exists + package_json_path = os.path.join(self.state_manager.get_full_project_root(), "package.json") + if not os.path.exists(package_json_path): + # Skip if no package.json found + return + + # Then check if node_modules directory exists + node_modules_path = os.path.join(self.state_manager.get_full_project_root(), "node_modules") + if not os.path.exists(node_modules_path): + await self.send_message("Installing project dependencies...") + await self.process_manager.run_command("npm install", show_output=False) + + def handle_parallel_responses(self, agent: BaseAgent, responses: List[AgentResponse]) -> AgentResponse: + """ + Handle responses from agents that were run in parallel. + + This method is called when multiple agents are run in parallel, and it + should return a single response that represents the combined responses + of all agents. + + :param agent: The original agent that was run in parallel. + :param responses: List of responses from all agents. + :return: Combined response. + """ + response = AgentResponse.done(agent) + if isinstance(agent, CodeMonkey): + files = [] + for single_response in responses: + if single_response.type == ResponseType.INPUT_REQUIRED: + files += single_response.data.get("files", []) + break + if files: + response = AgentResponse.input_required(agent, files) + return response + else: + raise ValueError(f"Unhandled parallel agent type: {agent.__class__.__name__}") + + async def offline_changes_check(self): + """ + Check for changes outside Pythagora. + + If there are changes, ask the user if they want to keep them, and + import if needed. + """ + + log.info("Checking for offline changes.") + modified_files = await self.state_manager.get_modified_files_with_content() + + if self.state_manager.workspace_is_empty(): + # NOTE: this will currently get triggered on a new project, but will do + # nothing as there's no files in the database. + log.info("Detected empty workspace, restoring state from the database.") + await self.state_manager.restore_files() + elif modified_files: + await self.send_message(f"We found {len(modified_files)} new and/or modified files.") + await self.ui.send_modified_files(modified_files) + hint = "".join( + [ + "If you would like Pythagora to import those changes, click 'Yes'.\n", + "Clicking 'No' means Pythagora will restore (overwrite) all files to the last stored state.\n", + ] + ) + use_changes = await self.ask_question( + question="Would you like to keep your changes?", + buttons={ + "yes": "Yes, keep my changes", + "no": "No, restore last Pythagora state", + }, + buttons_only=True, + hint=hint, + ) + if use_changes.button == "yes": + log.debug("Importing offline changes into Pythagora.") + await self.import_files() + else: + log.debug("Restoring last stored state.") + await self.state_manager.restore_files() + + log.info("Offline changes check done.") + + async def handle_done(self, agent: BaseAgent, response: AgentResponse) -> AgentResponse: + """ + Handle the DONE response from the agent and commit current state to the database. + + This also checks for any files created or modified outside Pythagora and + imports them. If any of the files require input from the user, the returned response + will trigger the HumanInput agent to ask the user to provide the required input. + + """ + n_epics = len(self.next_state.epics) + n_finished_epics = n_epics - len(self.next_state.unfinished_epics) + n_tasks = len(self.next_state.tasks) + n_finished_tasks = n_tasks - len(self.next_state.unfinished_tasks) + n_iterations = len(self.next_state.iterations) + n_finished_iterations = n_iterations - len(self.next_state.unfinished_iterations) + n_steps = len(self.next_state.steps) + n_finished_steps = n_steps - len(self.next_state.unfinished_steps) + + log.debug( + f"Agent {agent.__class__.__name__} is done, " + f"committing state for step {self.current_state.step_index}: " + f"{n_finished_epics}/{n_epics} epics, " + f"{n_finished_tasks}/{n_tasks} tasks, " + f"{n_finished_iterations}/{n_iterations} iterations, " + f"{n_finished_steps}/{n_steps} dev steps." + ) + await self.state_manager.commit() + + # If there are any new or modified files changed outside Pythagora, + # this is a good time to add them to the project. If any of them have + # INPUT_REQUIRED, we'll first ask the user to provide the required input. + import_files_response = await self.import_files() + + # If any of the files are missing metadata/descriptions, those need to be filled-in + missing_descriptions = [file.path for file in self.current_state.files if not file.meta.get("description")] + if missing_descriptions: + log.debug(f"Some files are missing descriptions: {', '.join(missing_descriptions)}, requesting analysis") + return AgentResponse.describe_files(self) + + return import_files_response + + def create_agent(self, prev_response: Optional[AgentResponse]) -> Union[List[BaseAgent], BaseAgent]: + state = self.current_state + + if prev_response: + if prev_response.type in [ResponseType.CANCEL, ResponseType.ERROR]: + return ErrorHandler(self.state_manager, self.ui, prev_response=prev_response) + if prev_response.type == ResponseType.DESCRIBE_FILES: + return CodeMonkey(self.state_manager, self.ui, prev_response=prev_response) + if prev_response.type == ResponseType.INPUT_REQUIRED: + # FIXME: HumanInput should be on the whole time and intercept chat/interrupt + return HumanInput(self.state_manager, self.ui, prev_response=prev_response) + if prev_response.type == ResponseType.IMPORT_PROJECT: + return Importer(self.state_manager, self.ui, prev_response=prev_response) + if prev_response.type == ResponseType.EXTERNAL_DOCS_REQUIRED: + return ExternalDocumentation(self.state_manager, self.ui, prev_response=prev_response) + if prev_response.type == ResponseType.UPDATE_SPECIFICATION: + return SpecWriter(self.state_manager, self.ui, prev_response=prev_response) + + if not state.epics or (state.current_epic and state.current_epic.get("source") == "frontend"): + # Build frontend + return Frontend(self.state_manager, self.ui, process_manager=self.process_manager) + elif not state.specification.description: + # New project: ask the Spec Writer to refine and save the project specification + return SpecWriter(self.state_manager, self.ui, process_manager=self.process_manager) + elif not state.specification.architecture: + # Ask the Architect to design the project architecture and determine dependencies + return Architect(self.state_manager, self.ui, process_manager=self.process_manager) + elif not self.current_state.unfinished_tasks or (state.specification.templates and not state.files): + # Ask the Tech Lead to break down the initial project or feature into tasks and apply project templates + return TechLead(self.state_manager, self.ui, process_manager=self.process_manager) + + # Current task status must be checked before Developer is called because we might want + # to skip it instead of breaking it down + current_task_status = state.current_task.get("status") if state.current_task else None + if current_task_status: + # Status of the current task is set first time after the task was reviewed by user + log.info(f"Status of current task: {current_task_status}") + if current_task_status == TaskStatus.REVIEWED: + # User reviewed the task, call TechnicalWriter to see if documentation needs to be updated + return TechnicalWriter(self.state_manager, self.ui) + elif current_task_status in [TaskStatus.DOCUMENTED, TaskStatus.SKIPPED]: + # Task is fully done or skipped, call TaskCompleter to mark it as completed + return TaskCompleter(self.state_manager, self.ui, process_manager=self.process_manager) + + if not state.steps and not state.iterations: + # Ask the Developer to break down current task into actionable steps + return Developer(self.state_manager, self.ui) + + if state.current_step: + # Execute next step in the task + # TODO: this can be parallelized in the future + return self.create_agent_for_step(state.current_step) + + if state.unfinished_iterations: + current_iteration_status = state.current_iteration["status"] + if current_iteration_status == IterationStatus.HUNTING_FOR_BUG: + # Triggering the bug hunter to start the hunt + return BugHunter(self.state_manager, self.ui) + elif current_iteration_status == IterationStatus.START_PAIR_PROGRAMMING: + # Pythagora cannot solve the issue so we're starting pair programming + return BugHunter(self.state_manager, self.ui) + elif current_iteration_status == IterationStatus.AWAITING_LOGGING: + # Get the developer to implement logs needed for debugging + return Developer(self.state_manager, self.ui) + elif current_iteration_status == IterationStatus.AWAITING_BUG_FIX: + # Get the developer to implement the bug fix for debugging + return Developer(self.state_manager, self.ui) + elif current_iteration_status == IterationStatus.IMPLEMENT_SOLUTION: + # Get the developer to implement the "change" requested by the user + return Developer(self.state_manager, self.ui) + elif current_iteration_status == IterationStatus.AWAITING_USER_TEST: + # Getting the bug hunter to ask the human to test the bug fix + return BugHunter(self.state_manager, self.ui) + elif current_iteration_status == IterationStatus.AWAITING_BUG_REPRODUCTION: + # Getting the bug hunter to ask the human to reproduce the bug + return BugHunter(self.state_manager, self.ui) + elif current_iteration_status == IterationStatus.FIND_SOLUTION: + # Find solution to the iteration problem + return Troubleshooter(self.state_manager, self.ui) + elif current_iteration_status == IterationStatus.PROBLEM_SOLVER: + # Call Problem Solver if the user said "I'm stuck in a loop" + return ProblemSolver(self.state_manager, self.ui) + elif current_iteration_status == IterationStatus.NEW_FEATURE_REQUESTED: + # Call Spec Writer to add the "change" requested by the user to project specification + return SpecWriter(self.state_manager, self.ui) + + # We have just finished the task, call Troubleshooter to ask the user to review + return Troubleshooter(self.state_manager, self.ui) + + def create_agent_for_step(self, step: dict) -> Union[List[BaseAgent], BaseAgent]: + step_type = step.get("type") + if step_type == "save_file": + steps = self.current_state.get_steps_of_type("save_file") + parallel = [] + for step in steps: + parallel.append(CodeMonkey(self.state_manager, self.ui, step=step)) + return parallel + elif step_type == "command": + return self.executor.for_step(step) + elif step_type == "human_intervention": + return HumanInput(self.state_manager, self.ui, step=step) + elif step_type == "review_task": + return LegacyHandler(self.state_manager, self.ui, data={"type": "review_task"}) + elif step_type == "create_readme": + return TechnicalWriter(self.state_manager, self.ui) + elif step_type == "utility_function": + return Developer(self.state_manager, self.ui) + else: + raise ValueError(f"Unknown step type: {step_type}") + + async def import_files(self) -> Optional[AgentResponse]: + imported_files, removed_paths = await self.state_manager.import_files() + if not imported_files and not removed_paths: + return None + + if imported_files: + log.info(f"Imported new/changed files to project: {', '.join(f.path for f in imported_files)}") + if removed_paths: + log.info(f"Removed files from project: {', '.join(removed_paths)}") + + input_required_files: list[dict[str, int]] = [] + for file in imported_files: + for line in self.state_manager.get_input_required(file.content.content, file.path): + input_required_files.append({"file": file.path, "line": line}) + + if input_required_files: + # This will trigger the HumanInput agent to ask the user to provide the required changes + # If the user changes anything (removes the "required changes"), the file will be re-imported. + return AgentResponse.input_required(self, input_required_files) + + # Commit the newly imported file + log.debug(f"Committing imported/removed files as a separate step {self.current_state.step_index}") + await self.state_manager.commit() + return None + + async def init_ui(self): + await self.ui.send_project_root(self.state_manager.get_full_project_root()) + await self.ui.loading_finished() + + if self.current_state.epics: + if len(self.current_state.epics) > 3: + # We only want to send previous features, ie. exclude current one and the initial project (first epic) + await self.ui.send_features_list([e["description"] for e in self.current_state.epics[2:-1]]) + + if self.current_state.specification.description: + await self.ui.send_project_description(self.current_state.specification.description) + + async def update_stats(self): + if self.current_state.steps and self.current_state.current_step: + source = self.current_state.current_step.get("source") + source_steps = self.current_state.get_last_iteration_steps() + await self.ui.send_step_progress( + source_steps.index(self.current_state.current_step) + 1, + len(source_steps), + self.current_state.current_step, + source, + ) + + total_files = 0 + total_lines = 0 + for file in self.current_state.files: + total_files += 1 + total_lines += len(file.content.content.splitlines()) + + telemetry.set("num_files", total_files) + telemetry.set("num_lines", total_lines) + + stats = telemetry.get_project_stats() + await self.ui.send_project_stats(stats) diff --git a/core/agents/problem_solver.py b/core/agents/problem_solver.py new file mode 100644 index 00000000..e7b8de05 --- /dev/null +++ b/core/agents/problem_solver.py @@ -0,0 +1,128 @@ +from typing import Optional + +from pydantic import BaseModel, Field + +from core.agents.base import BaseAgent +from core.agents.convo import AgentConvo +from core.agents.response import AgentResponse +from core.agents.troubleshooter import IterationPromptMixin +from core.db.models.project_state import IterationStatus +from core.llm.parser import JSONParser +from core.log import get_logger + +log = get_logger(__name__) + + +class AlternativeSolutions(BaseModel): + # FIXME: This is probably extra leftover from some dead code in the old implementation + description_of_tried_solutions: str = Field( + description="A description of the solutions that were tried to solve the recurring issue that was labeled as loop by the user.", + ) + alternative_solutions: list[str] = Field( + description=("List of all alternative solutions to the recurring issue that was labeled as loop by the user.") + ) + + +class ProblemSolver(IterationPromptMixin, BaseAgent): + agent_type = "problem-solver" + display_name = "Problem Solver" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.iteration = self.current_state.current_iteration + self.next_state_iteration = self.next_state.current_iteration + self.previous_solutions = [s for s in self.iteration["alternative_solutions"] if s["tried"]] + self.possible_solutions = [s for s in self.iteration["alternative_solutions"] if not s["tried"]] + + async def run(self) -> AgentResponse: + if self.iteration is None: + log.warning("ProblemSolver agent started without an iteration to work on, possible bug?") + return AgentResponse.done(self) + + if not self.possible_solutions: + await self.generate_alternative_solutions() + return AgentResponse.done(self) + + return await self.try_alternative_solutions() + + async def generate_alternative_solutions(self): + llm = self.get_llm(stream_output=True) + convo = ( + AgentConvo(self) + .template( + "get_alternative_solutions", + user_input=self.iteration["user_feedback"], + iteration=self.iteration, + previous_solutions=self.previous_solutions, + ) + .require_schema(AlternativeSolutions) + ) + llm_response: AlternativeSolutions = await llm( + convo, + parser=JSONParser(spec=AlternativeSolutions), + temperature=1, + ) + self.next_state_iteration["alternative_solutions"] = self.iteration["alternative_solutions"] + [ + { + "user_feedback": None, + "description": solution, + "tried": False, + } + for solution in llm_response.alternative_solutions + ] + self.next_state.flag_iterations_as_modified() + + async def try_alternative_solutions(self) -> AgentResponse: + preferred_solution = await self.ask_for_preferred_solution() + if preferred_solution is None: + # TODO: We have several alternative solutions but the user didn't choose any. + # This means the user either needs expert help, or that they need to go back and + # maybe rephrase the tasks or even the project specs. + # For now, we'll just mark these as not working and try to regenerate. + self.next_state_iteration["alternative_solutions"] = [ + { + **s, + "tried": True, + "user_feedback": s["user_feedback"] or "That doesn't sound like a good idea, try something else.", + } + for s in self.possible_solutions + ] + self.next_state.flag_iterations_as_modified() + return AgentResponse.done(self) + + index, next_solution_to_try = preferred_solution + llm_solution = await self.find_solution( + self.iteration["user_feedback"], + next_solution_to_try=next_solution_to_try, + ) + + self.next_state_iteration["alternative_solutions"][index]["tried"] = True + self.next_state_iteration["description"] = llm_solution + self.next_state_iteration["attempts"] = self.iteration["attempts"] + 1 + self.next_state_iteration["status"] = IterationStatus.PROBLEM_SOLVER + self.next_state.flag_iterations_as_modified() + return AgentResponse.done(self) + + async def ask_for_preferred_solution(self) -> Optional[tuple[int, str]]: + solutions = self.possible_solutions + buttons = {} + + for i in range(len(solutions)): + buttons[str(i)] = str(i + 1) + buttons["none"] = "None of these" + + solutions_txt = "\n\n".join([f"{i+1}: {s['description']}" for i, s in enumerate(solutions)]) + user_response = await self.ask_question( + "Choose which solution would you like Pythagora to try next:\n\n" + solutions_txt, + buttons=buttons, + default="0", + buttons_only=True, + ) + if user_response.button == "none" or user_response.cancelled: + return None + + try: + i = int(user_response.button) + return i, solutions[i] + except (ValueError, IndexError): + return None diff --git a/core/agents/response.py b/core/agents/response.py new file mode 100644 index 00000000..7a40a8be --- /dev/null +++ b/core/agents/response.py @@ -0,0 +1,100 @@ +from enum import Enum +from typing import TYPE_CHECKING, Optional + +from core.log import get_logger + +if TYPE_CHECKING: + from core.agents.base import BaseAgent + from core.agents.error_handler import ErrorHandler + + +log = get_logger(__name__) + + +class ResponseType(str, Enum): + DONE = "done" + """Agent has finished processing.""" + + ERROR = "error" + """There was an error processing the request.""" + + CANCEL = "cancel" + """User explicitly cancelled the operation.""" + + EXIT = "exit" + """Pythagora should exit.""" + + DESCRIBE_FILES = "describe-files" + """Analysis of the files in the project is requested.""" + + INPUT_REQUIRED = "input-required" + """User needs to modify a line in the generated code.""" + + IMPORT_PROJECT = "import-project" + """User wants to import an existing project.""" + + EXTERNAL_DOCS_REQUIRED = "external-docs-required" + """We need to fetch external docs for a task.""" + + UPDATE_SPECIFICATION = "update-specification" + """We need to update the project specification.""" + + +class AgentResponse: + type: ResponseType = ResponseType.DONE + agent: "BaseAgent" + data: Optional[dict] + + def __init__(self, type: ResponseType, agent: "BaseAgent", data: Optional[dict] = None): + self.type = type + self.agent = agent + self.data = data + + def __repr__(self) -> str: + return f"" + + @staticmethod + def done(agent: "BaseAgent") -> "AgentResponse": + return AgentResponse(type=ResponseType.DONE, agent=agent) + + @staticmethod + def error(agent: "BaseAgent", message: str, details: Optional[dict] = None) -> "AgentResponse": + return AgentResponse( + type=ResponseType.ERROR, + agent=agent, + data={"message": message, "details": details}, + ) + + @staticmethod + def cancel(agent: "BaseAgent") -> "AgentResponse": + return AgentResponse(type=ResponseType.CANCEL, agent=agent) + + @staticmethod + def exit(agent: "ErrorHandler") -> "AgentResponse": + return AgentResponse(type=ResponseType.EXIT, agent=agent) + + @staticmethod + def describe_files(agent: "BaseAgent") -> "AgentResponse": + return AgentResponse(type=ResponseType.DESCRIBE_FILES, agent=agent) + + @staticmethod + def input_required(agent: "BaseAgent", files: list[dict[str, int]]) -> "AgentResponse": + return AgentResponse(type=ResponseType.INPUT_REQUIRED, agent=agent, data={"files": files}) + + @staticmethod + def import_project(agent: "BaseAgent") -> "AgentResponse": + return AgentResponse(type=ResponseType.IMPORT_PROJECT, agent=agent) + + @staticmethod + def external_docs_required(agent: "BaseAgent") -> "AgentResponse": + return AgentResponse(type=ResponseType.EXTERNAL_DOCS_REQUIRED, agent=agent) + + @staticmethod + def update_specification(agent: "BaseAgent", description: str) -> "AgentResponse": + return AgentResponse( + type=ResponseType.UPDATE_SPECIFICATION, + agent=agent, + data={ + "description": description, + }, + ) diff --git a/core/agents/spec_writer.py b/core/agents/spec_writer.py new file mode 100644 index 00000000..b3ba374c --- /dev/null +++ b/core/agents/spec_writer.py @@ -0,0 +1,208 @@ +from core.agents.base import BaseAgent +from core.agents.convo import AgentConvo +from core.agents.response import AgentResponse, ResponseType +from core.config import SPEC_WRITER_AGENT_NAME +from core.db.models import Complexity +from core.db.models.project_state import IterationStatus +from core.llm.parser import StringParser +from core.log import get_logger +from core.telemetry import telemetry + +# If the project description is less than this, perform an analysis using LLM +ANALYZE_THRESHOLD = 1500 +# URL to the wiki page with tips on how to write a good project description +INITIAL_PROJECT_HOWTO_URL = ( + "https://github.com/Pythagora-io/gpt-pilot/wiki/How-to-write-a-good-initial-project-description" +) +SPEC_STEP_NAME = "Create specification" + +log = get_logger(__name__) + + +class SpecWriter(BaseAgent): + agent_type = "spec-writer" + display_name = "Spec Writer" + + async def run(self) -> AgentResponse: + current_iteration = self.current_state.current_iteration + if current_iteration is not None and current_iteration.get("status") == IterationStatus.NEW_FEATURE_REQUESTED: + return await self.update_spec(iteration_mode=True) + elif self.prev_response and self.prev_response.type == ResponseType.UPDATE_SPECIFICATION: + return await self.update_spec(iteration_mode=False) + else: + return await self.initialize_spec() + + async def initialize_spec(self) -> AgentResponse: + # response = await self.ask_question( + # "Describe your app in as much detail as possible", + # allow_empty=False, + # ) + # if response.cancelled: + # return AgentResponse.error(self, "No project description") + # + # user_description = response.text.strip() + user_description = self.current_state.epics[0]["description"] + + complexity = await self.check_prompt_complexity(user_description) + await telemetry.trace_code_event( + "project-description", + { + "initial_prompt": user_description, + "complexity": complexity, + }, + ) + + reviewed_spec = user_description + # if len(user_description) < ANALYZE_THRESHOLD and complexity != Complexity.SIMPLE: + # initial_spec = await self.analyze_spec(user_description) + # reviewed_spec = await self.review_spec(desc=user_description, spec=initial_spec) + + self.next_state.specification = self.current_state.specification.clone() + self.next_state.specification.original_description = user_description + self.next_state.specification.description = reviewed_spec + self.next_state.specification.complexity = complexity + telemetry.set("initial_prompt", user_description) + telemetry.set("updated_prompt", reviewed_spec) + telemetry.set("is_complex_app", complexity != Complexity.SIMPLE) + + self.next_state.action = SPEC_STEP_NAME + return AgentResponse.done(self) + + async def update_spec(self, iteration_mode) -> AgentResponse: + if iteration_mode: + feature_description = self.current_state.current_iteration["user_feedback"] + else: + feature_description = self.prev_response.data["description"] + + await self.send_message( + f"Making the following changes to project specification:\n\n{feature_description}\n\nUpdated project specification:" + ) + llm = self.get_llm(SPEC_WRITER_AGENT_NAME, stream_output=True) + convo = AgentConvo(self).template("add_new_feature", feature_description=feature_description) + llm_response: str = await llm(convo, temperature=0, parser=StringParser()) + updated_spec = llm_response.strip() + await self.ui.generate_diff( + "project_specification", self.current_state.specification.description, updated_spec, source=self.ui_source + ) + user_response = await self.ask_question( + "Do you accept these changes to the project specification?", + buttons={"yes": "Yes", "no": "No"}, + default="yes", + buttons_only=True, + ) + await self.ui.close_diff() + + if user_response.button == "yes": + self.next_state.specification = self.current_state.specification.clone() + self.next_state.specification.description = updated_spec + telemetry.set("updated_prompt", updated_spec) + + if iteration_mode: + self.next_state.current_iteration["status"] = IterationStatus.FIND_SOLUTION + self.next_state.flag_iterations_as_modified() + else: + complexity = await self.check_prompt_complexity(feature_description) + self.next_state.current_epic["complexity"] = complexity + + return AgentResponse.done(self) + + async def check_prompt_complexity(self, prompt: str) -> str: + is_feature = self.current_state.epics and len(self.current_state.epics) > 2 + await self.send_message("Checking the complexity of the prompt ...") + llm = self.get_llm(SPEC_WRITER_AGENT_NAME) + convo = AgentConvo(self).template( + "prompt_complexity", + prompt=prompt, + is_feature=is_feature, + ) + llm_response: str = await llm(convo, temperature=0, parser=StringParser()) + log.info(f"Complexity check response: {llm_response}") + return llm_response.lower() + + async def analyze_spec(self, spec: str) -> str: + msg = ( + "Your project description seems a bit short. " + "The better you can describe the project, the better Pythagora will understand what you'd like to build.\n\n" + f"Here are some tips on how to better describe the project: {INITIAL_PROJECT_HOWTO_URL}\n\n" + "Let's start by refining your project idea:" + ) + await self.send_message(msg) + + llm = self.get_llm(SPEC_WRITER_AGENT_NAME, stream_output=True) + convo = AgentConvo(self).template("ask_questions").user(spec) + n_questions = 0 + n_answers = 0 + + while True: + response: str = await llm(convo) + if len(response) > 500: + # The response is too long for it to be a question, assume it's the updated spec + confirm = await self.ask_question( + ("Would you like to change or add anything? Write it out here."), + allow_empty=True, + buttons={"continue": "No thanks, the spec looks good"}, + ) + if confirm.cancelled or confirm.button == "continue" or confirm.text == "": + updated_spec = response.strip() + await telemetry.trace_code_event( + "spec-writer-questions", + { + "num_questions": n_questions, + "num_answers": n_answers, + "new_spec": updated_spec, + }, + ) + return updated_spec + convo.user(confirm.text) + + else: + convo.assistant(response) + + n_questions += 1 + user_response = await self.ask_question( + response, + buttons={"skip": "Skip this question", "skip_all": "No more questions"}, + verbose=False, + ) + if user_response.cancelled or user_response.button == "skip_all": + convo.user( + "This is enough clarification, you have all the information. " + "Please output the spec now, without additional comments or questions." + ) + response: str = await llm(convo) + confirm = await self.ask_question( + ("Would you like to change or add anything? Write it out here."), + allow_empty=True, + buttons={"continue": "No thanks, the spec looks good"}, + ) + if confirm.cancelled or confirm.button == "continue" or confirm.text == "": + updated_spec = response.strip() + await telemetry.trace_code_event( + "spec-writer-questions", + { + "num_questions": n_questions, + "num_answers": n_answers, + "new_spec": updated_spec, + }, + ) + return updated_spec + convo.user(confirm.text) + continue + + n_answers += 1 + if user_response.button == "skip": + convo.user("Skip this question.") + continue + else: + convo.user(user_response.text) + + async def review_spec(self, desc: str, spec: str) -> str: + convo = AgentConvo(self).template("review_spec", desc=desc, spec=spec) + llm = self.get_llm(SPEC_WRITER_AGENT_NAME) + llm_response: str = await llm(convo, temperature=0) + additional_info = llm_response.strip() + if additional_info and len(additional_info) > 6: + spec += "\n\nAdditional info/examples:\n\n" + additional_info + await self.send_message(f"\n\nAdditional info/examples:\n\n {additional_info}") + + return spec diff --git a/core/agents/task_completer.py b/core/agents/task_completer.py new file mode 100644 index 00000000..7f00ad77 --- /dev/null +++ b/core/agents/task_completer.py @@ -0,0 +1,57 @@ +from core.agents.base import BaseAgent +from core.agents.git import GitMixin +from core.agents.response import AgentResponse +from core.log import get_logger +from core.telemetry import telemetry + +log = get_logger(__name__) + + +class TaskCompleter(BaseAgent, GitMixin): + agent_type = "pythagora" + display_name = "Pythagora" + + async def run(self) -> AgentResponse: + if self.state_manager.git_available and self.state_manager.git_used: + await self.git_commit() + + current_task_index1 = self.current_state.tasks.index(self.current_state.current_task) + 1 + self.next_state.action = f"Task #{current_task_index1} complete" + self.next_state.complete_task() + await self.state_manager.log_task_completed() + tasks = self.current_state.tasks + source = self.current_state.current_epic.get("source", "app") + await self.ui.send_task_progress( + current_task_index1, + len(tasks), + self.current_state.current_task["description"], + source, + "done", + self.current_state.get_source_index(source), + tasks, + ) + await telemetry.trace_code_event( + "task-end", + { + "task_index": current_task_index1, + "num_tasks": len(self.current_state.tasks), + "num_epics": len(self.current_state.epics), + "num_iterations": len(self.current_state.iterations), + }, + ) + + if current_task_index1 == len(tasks): + if source == "app": + await self.ui.send_app_finished( + app_id=str(self.state_manager.project.id), + app_name=self.state_manager.project.name, + folder_name=self.state_manager.project.folder_name, + ) + elif source == "feature": + await self.ui.send_feature_finished( + app_id=str(self.state_manager.project.id), + app_name=self.state_manager.project.name, + folder_name=self.state_manager.project.folder_name, + ) + + return AgentResponse.done(self) diff --git a/core/agents/tech_lead.py b/core/agents/tech_lead.py new file mode 100644 index 00000000..d0211770 --- /dev/null +++ b/core/agents/tech_lead.py @@ -0,0 +1,319 @@ +import json +from uuid import uuid4 + +from pydantic import BaseModel, Field + +from core.agents.base import BaseAgent +from core.agents.convo import AgentConvo +from core.agents.mixins import RelevantFilesMixin +from core.agents.response import AgentResponse +from core.config import TECH_LEAD_EPIC_BREAKDOWN, TECH_LEAD_PLANNING +from core.db.models import Complexity +from core.db.models.project_state import TaskStatus +from core.llm.parser import JSONParser +from core.log import get_logger +from core.telemetry import telemetry +from core.templates.registry import PROJECT_TEMPLATES +from core.ui.base import ProjectStage, pythagora_source, success_source + +log = get_logger(__name__) + + +class APIEndpoint(BaseModel): + description: str = Field(description="Description of an API endpoint.") + method: str = Field(description="HTTP method of the API endpoint.") + endpoint: str = Field(description="URL of the API endpoint.") + request_body: dict = Field(description="Request body of the API endpoint.") + response_body: dict = Field(description="Response body of the API endpoint.") + + +class Epic(BaseModel): + description: str = Field(description="Description of an epic.") + + +class Task(BaseModel): + description: str = Field(description="Description of a task.") + related_api_endpoints: list[APIEndpoint] = Field(description="API endpoints that will be implemented in this task.") + testing_instructions: str = Field(description="Instructions for testing the task.") + + +class DevelopmentPlan(BaseModel): + plan: list[Epic] = Field(description="List of epics that need to be done to implement the entire plan.") + + +class EpicPlan(BaseModel): + plan: list[Task] = Field(description="List of tasks that need to be done to implement the entire epic.") + + +class TechLead(RelevantFilesMixin, BaseAgent): + agent_type = "tech-lead" + display_name = "Tech Lead" + + async def run(self) -> AgentResponse: + # Building frontend is the first epic + if len(self.current_state.epics) == 1: + self.create_initial_project_epic() + return AgentResponse.done(self) + + # if self.current_state.specification.templates and len(self.current_state.files) < 2: + # await self.apply_project_templates() + # self.next_state.action = "Apply project templates" + # await self.ui.send_epics_and_tasks( + # self.next_state.current_epic["sub_epics"], + # self.next_state.tasks, + # ) + # + # inputs = [] + # for file in self.next_state.files: + # input_required = self.state_manager.get_input_required(file.content.content) + # if input_required: + # inputs += [{"file": file.path, "line": line} for line in input_required] + # + # if inputs: + # return AgentResponse.input_required(self, inputs) + # else: + # return AgentResponse.done(self) + + if self.current_state.current_epic: + self.next_state.action = "Create a development plan" + return await self.plan_epic(self.current_state.current_epic) + else: + return await self.ask_for_new_feature() + + def create_initial_project_epic(self): + log.debug("Creating initial project Epic") + self.next_state.epics = self.current_state.epics + [ + { + "id": uuid4().hex, + "name": "Initial Project", + "source": "app", + "description": self.current_state.specification.description, + "test_instructions": None, + "summary": None, + "completed": False, + "complexity": self.current_state.specification.complexity, + "sub_epics": [], + } + ] + self.next_state.relevant_files = None + self.next_state.modified_files = {} + + async def apply_project_templates(self): + state = self.current_state + summaries = [] + + # Only do this for the initial project and if the templates are specified + if len(state.epics) != 1 or not state.specification.templates: + return + + for template_name, template_options in state.specification.templates.items(): + template_class = PROJECT_TEMPLATES.get(template_name) + if not template_class: + log.error(f"Project template not found: {template_name}") + continue + + template = template_class( + template_options, + self.state_manager, + self.process_manager, + ) + + description = template.description + log.info(f"Applying project template: {template.name}") + await self.send_message(f"Applying project template {description} ...") + summary = await template.apply() + summaries.append(summary) + + # Saving template files will fill this in and we want it clear for the first task. + self.next_state.relevant_files = None + + if summaries: + spec = self.current_state.specification.clone() + spec.template_summary = "\n\n".join(summaries) + + self.next_state.specification = spec + + async def ask_for_new_feature(self) -> AgentResponse: + if len(self.current_state.epics) > 2: + await self.ui.send_message("Your new feature is complete!", source=success_source) + else: + await self.ui.send_message("Your app is DONE! You can start using it right now!", source=success_source) + + if self.current_state.run_command: + await self.ui.send_run_command(self.current_state.run_command) + + log.debug("Asking for new feature") + response = await self.ask_question( + "Do you have a new feature to add to the project? Just write it here:", + buttons={"continue": "continue", "end": "No, I'm done"}, + allow_empty=False, + extra_info="restart_app", + ) + + if response.button == "end" or response.cancelled or not response.text: + await self.ui.send_message("Thank you for using Pythagora!", source=pythagora_source) + return AgentResponse.exit(self) + + feature_description = response.text + self.next_state.epics = self.current_state.epics + [ + { + "id": uuid4().hex, + "name": f"Feature #{len(self.current_state.epics)}", + "test_instructions": None, + "source": "feature", + "description": feature_description, + "summary": None, + "completed": False, + "complexity": None, # Determined and defined in SpecWriter + "sub_epics": [], + } + ] + # Orchestrator will rerun us to break down the new feature epic + self.next_state.action = f"Start of feature #{len(self.current_state.epics)}" + return AgentResponse.update_specification(self, feature_description) + + async def plan_epic(self, epic) -> AgentResponse: + log.debug(f"Planning tasks for the epic: {epic['name']}") + await self.send_message("Creating the development plan ...") + + if epic.get("source") == "feature": + await self.get_relevant_files(user_feedback=epic.get("description")) + + llm = self.get_llm(TECH_LEAD_PLANNING) + convo = ( + AgentConvo(self) + .template( + "plan", + epic=epic, + task_type=self.current_state.current_epic.get("source", "app"), + # FIXME: we're injecting summaries to initial description + existing_summary=None, + get_only_api_files=True, + ) + .require_schema(DevelopmentPlan) + ) + + response: DevelopmentPlan = await llm(convo, parser=JSONParser(DevelopmentPlan)) + + convo.remove_last_x_messages(1) + formatted_epics = [f"Epic #{index}: {epic.description}" for index, epic in enumerate(response.plan, start=1)] + epics_string = "\n\n".join(formatted_epics) + convo = convo.assistant(epics_string) + llm = self.get_llm(TECH_LEAD_EPIC_BREAKDOWN) + + if epic.get("source") == "feature" or epic.get("complexity") == Complexity.SIMPLE: + await self.send_message(f"Epic 1: {epic['name']}") + self.next_state.current_epic["sub_epics"] = [ + { + "id": 1, + "description": epic["name"], + } + ] + await self.send_message("Creating tasks for this epic ...") + self.next_state.tasks = self.next_state.tasks + [ + { + "id": uuid4().hex, + "description": task.description, + "instructions": None, + "pre_breakdown_testing_instructions": None, + "status": TaskStatus.TODO, + "sub_epic_id": 1, + } + for task in response.plan + ] + else: + self.next_state.current_epic["sub_epics"] = [ + { + "id": sub_epic_number, + "description": sub_epic.description, + } + for sub_epic_number, sub_epic in enumerate(response.plan, start=1) + ] + for sub_epic_number, sub_epic in enumerate(response.plan, start=1): + await self.send_message(f"Epic {sub_epic_number}: {sub_epic.description}") + convo = convo.template( + "epic_breakdown", + epic_number=sub_epic_number, + epic_description=sub_epic.description, + get_only_api_files=True, + ).require_schema(EpicPlan) + await self.send_message("Creating tasks for this epic ...") + epic_plan: EpicPlan = await llm(convo, parser=JSONParser(EpicPlan)) + self.next_state.tasks = self.next_state.tasks + [ + { + "id": uuid4().hex, + "description": task.description, + "instructions": None, + "pre_breakdown_testing_instructions": task.testing_instructions, + "status": TaskStatus.TODO, + "sub_epic_id": sub_epic_number, + "related_api_endpoints": [rae.model_dump() for rae in (task.related_api_endpoints or [])], + } + for task in epic_plan.plan + ] + convo.remove_last_x_messages(2) + + await self.ui.send_epics_and_tasks( + self.next_state.current_epic["sub_epics"], + self.next_state.tasks, + ) + + await self.ui.send_project_stage({"stage": ProjectStage.OPEN_PLAN}) + response = await self.ask_question( + "Open and edit your development plan in the Progress tab", + buttons={"done_editing": "I'm done editing, the plan looks good"}, + default="done_editing", + buttons_only=True, + extra_info="edit_plan", + ) + + self.update_epics_and_tasks(response.text) + + await self.ui.send_epics_and_tasks( + self.next_state.current_epic["sub_epics"], + self.next_state.tasks, + ) + + await telemetry.trace_code_event( + "development-plan", + { + "num_tasks": len(self.current_state.tasks), + "num_epics": len(self.current_state.epics), + }, + ) + return AgentResponse.done(self) + + def update_epics_and_tasks(self, edited_plan_string): + edited_plan = json.loads(edited_plan_string) + updated_tasks = [] + + existing_tasks_map = {task["description"]: task for task in self.next_state.tasks} + + self.next_state.current_epic["sub_epics"] = [] + for sub_epic_number, sub_epic in enumerate(edited_plan, start=1): + self.next_state.current_epic["sub_epics"].append( + { + "id": sub_epic_number, + "description": sub_epic["description"], + } + ) + + for task in sub_epic["tasks"]: + original_task = existing_tasks_map.get(task["description"]) + if original_task and task == original_task: + updated_task = original_task.copy() + updated_task["sub_epic_id"] = sub_epic_number + updated_tasks.append(updated_task) + else: + updated_tasks.append( + { + "id": uuid4().hex, + "description": task["description"], + "instructions": None, + "pre_breakdown_testing_instructions": None, + "status": TaskStatus.TODO, + "sub_epic_id": sub_epic_number, + } + ) + + self.next_state.tasks = updated_tasks diff --git a/core/agents/tech_writer.py b/core/agents/tech_writer.py new file mode 100644 index 00000000..90d6f5ef --- /dev/null +++ b/core/agents/tech_writer.py @@ -0,0 +1,58 @@ +from core.agents.base import BaseAgent +from core.agents.convo import AgentConvo +from core.agents.response import AgentResponse +from core.db.models.project_state import TaskStatus +from core.log import get_logger +from core.ui.base import success_source + +log = get_logger(__name__) + + +class TechnicalWriter(BaseAgent): + agent_type = "tech-writer" + display_name = "Technical Writer" + + async def run(self) -> AgentResponse: + n_tasks = len(self.current_state.tasks) + # current task is still "unfinished" at this point but for purposes of this agent, we want to consider + # it as "finished" and that is why we are subtracting 1 from the total number of unfinished tasks + n_unfinished = len(self.current_state.unfinished_tasks) - 1 + + if n_unfinished in [n_tasks // 2, 1]: + # Halfway through the initial project, and at the last task + await self.send_congratulations() + await self.create_readme() + + self.next_state.action = "Create README.md" + self.next_state.set_current_task_status(TaskStatus.DOCUMENTED) + return AgentResponse.done(self) + + async def send_congratulations(self): + n_tasks = len(self.current_state.tasks) + if not n_tasks: + log.warning("No tasks found in the project") + return + + n_unfinished = len(self.current_state.unfinished_tasks) - 1 + n_finished = n_tasks - n_unfinished + pct_finished = int(n_finished / n_tasks * 100) + n_files = len(self.current_state.files) + n_lines = sum(len(f.content.content.splitlines()) for f in self.current_state.files) + await self.ui.send_message( + "\n\n".join( + [ + f"CONGRATULATIONS! You reached {pct_finished}% of your project generation!", + f"For now, you have created {n_files} files with a total of {n_lines} lines of code.", + "Before continuing, Pythagora will create some documentation for the project...", + ] + ), + source=success_source, + ) + + async def create_readme(self): + await self.send_message("Creating README ...") + + llm = self.get_llm(stream_output=True) + convo = AgentConvo(self).template("create_readme") + llm_response: str = await llm(convo) + await self.state_manager.save_file("README.md", llm_response) diff --git a/core/agents/troubleshooter.py b/core/agents/troubleshooter.py new file mode 100644 index 00000000..5f2c9ca3 --- /dev/null +++ b/core/agents/troubleshooter.py @@ -0,0 +1,395 @@ +import json +from typing import Optional +from uuid import uuid4 + +from pydantic import BaseModel, Field + +from core.agents.base import BaseAgent +from core.agents.convo import AgentConvo +from core.agents.mixins import ChatWithBreakdownMixin, IterationPromptMixin, RelevantFilesMixin, TestSteps +from core.agents.response import AgentResponse +from core.config import TROUBLESHOOTER_GET_RUN_COMMAND +from core.db.models.file import File +from core.db.models.project_state import IterationStatus, TaskStatus +from core.llm.parser import JSONParser, OptionalCodeBlockParser +from core.log import get_logger +from core.telemetry import telemetry +from core.ui.base import ProjectStage, pythagora_source + +log = get_logger(__name__) + +LOOP_THRESHOLD = 3 # number of iterations in task to be considered a loop + + +class BugReportQuestions(BaseModel): + missing_data: list[str] = Field( + description="Very clear question that needs to be answered to have good bug report." + ) + + +class RouteFilePaths(BaseModel): + files: list[str] = Field(description="List of paths for files that contain routes") + + +class Troubleshooter(ChatWithBreakdownMixin, IterationPromptMixin, RelevantFilesMixin, BaseAgent): + agent_type = "troubleshooter" + display_name = "Troubleshooter" + + async def run(self) -> AgentResponse: + if self.current_state.unfinished_iterations: + if self.current_state.current_iteration.get("status") == IterationStatus.FIND_SOLUTION: + return await self.propose_solution() + else: + raise ValueError("There is unfinished iteration but it's not in FIND_SOLUTION state.") + else: + return await self.create_iteration() + + async def propose_solution(self) -> AgentResponse: + user_feedback = self.current_state.current_iteration.get("user_feedback") + user_feedback_qa = self.current_state.current_iteration.get("user_feedback_qa") + bug_hunting_cycles = self.current_state.current_iteration.get("bug_hunting_cycles") + + llm_solution = await self.find_solution( + user_feedback, user_feedback_qa=user_feedback_qa, bug_hunting_cycles=bug_hunting_cycles + ) + + self.next_state.current_iteration["description"] = llm_solution + self.next_state.current_iteration["status"] = IterationStatus.IMPLEMENT_SOLUTION + self.next_state.flag_iterations_as_modified() + + return AgentResponse.done(self) + + async def create_iteration(self) -> AgentResponse: + run_command = await self.get_run_command() + + user_instructions = self.current_state.current_task.get("test_instructions") + if not user_instructions: + user_instructions = await self.get_user_instructions() + if user_instructions is None: + # LLM decided we don't need to test anything, so we're done with the task + return await self.complete_task() + + # Save the user instructions for future iterations and rerun + self.next_state.current_task["test_instructions"] = user_instructions + self.next_state.flag_tasks_as_modified() + return AgentResponse.done(self) + else: + await self.ui.send_project_stage({"stage": ProjectStage.TEST_APP}) + await self.ui.send_message("Test the app by following these steps:", source=pythagora_source) + + await self.send_message("") + await self.ui.stop_app() + await self.ui.send_test_instructions(user_instructions, project_state_id=str(self.current_state.id)) + + # Developer sets iteration as "completed" when it generates the step breakdown, so we can't + # use "current_iteration" here + last_iteration = self.current_state.iterations[-1] if len(self.current_state.iterations) >= 3 else None + + should_iterate, is_loop, bug_report, change_description = await self.get_user_feedback( + run_command, + user_instructions, + last_iteration is not None, + ) + if not should_iterate: + # User tested and reported no problems, we're done with the task + return await self.complete_task() + + user_feedback = bug_report or change_description + user_feedback_qa = None # await self.generate_bug_report(run_command, user_instructions, user_feedback) + + if is_loop: + if last_iteration is not None and last_iteration.get("alternative_solutions"): + # If we already have alternative solutions, it means we were already in a loop. + return self.try_next_alternative_solution(user_feedback, user_feedback_qa) + else: + # Newly detected loop + iteration_status = IterationStatus.PROBLEM_SOLVER + await self.trace_loop("loop-feedback") + elif bug_report is not None: + iteration_status = IterationStatus.HUNTING_FOR_BUG + else: + # should be - elif change_description is not None: - but to prevent bugs with the extension + # this might be caused if we show the input field instead of buttons + iteration_status = IterationStatus.NEW_FEATURE_REQUESTED + + self.next_state.iterations = self.current_state.iterations + [ + { + "id": uuid4().hex, + "user_feedback": user_feedback, + "user_feedback_qa": user_feedback_qa, + "description": None, + "alternative_solutions": [], + # FIXME - this is incorrect if this is a new problem; otherwise we could + # just count the iterations + "attempts": 1, + "status": iteration_status, + "bug_hunting_cycles": [], + } + ] + + self.next_state.flag_iterations_as_modified() + if len(self.next_state.iterations) == LOOP_THRESHOLD: + await self.trace_loop("loop-start") + + return AgentResponse.done(self) + + async def complete_task(self) -> AgentResponse: + """ + No more coding or user interaction needed for the current task, mark it as reviewed. + After this it goes to TechnicalWriter for documentation. + """ + if len(self.current_state.iterations) >= LOOP_THRESHOLD: + await self.trace_loop("loop-end") + + current_task_index1 = self.current_state.tasks.index(self.current_state.current_task) + 1 + self.next_state.action = f"Task #{current_task_index1} reviewed" + self.next_state.set_current_task_status(TaskStatus.REVIEWED) + return AgentResponse.done(self) + + def _get_task_convo(self) -> AgentConvo: + # FIXME: Current prompts reuse conversation from the developer so we have to resort to this + task = self.current_state.current_task + current_task_index = self.current_state.tasks.index(task) + + return ( + AgentConvo(self) + .template( + "breakdown", + task=task, + iteration=None, + current_task_index=current_task_index, + related_api_endpoints=task.get("related_api_endpoints", []), + ) + .assistant(self.current_state.current_task["instructions"]) + ) + + async def get_run_command(self) -> Optional[str]: + if self.current_state.run_command: + return self.current_state.run_command + + await self.send_message("Figuring out how to run the app ...") + + llm = self.get_llm(TROUBLESHOOTER_GET_RUN_COMMAND) + convo = self._get_task_convo().template("get_run_command") + + # Although the prompt is explicit about not using "```", LLM may still return it + llm_response: str = await llm(convo, temperature=0, parser=OptionalCodeBlockParser()) + if len(llm_response) < 5: + llm_response = "" + self.next_state.run_command = llm_response + return llm_response + + async def get_user_instructions(self) -> Optional[str]: + await self.send_message("Determining how to test the app ...") + + route_files = await self._get_route_files() + current_task = self.current_state.current_task + + llm = self.get_llm() + convo = ( + self._get_task_convo() + .template( + "define_user_review_goal", + task=current_task, + route_files=route_files, + current_task_index=self.current_state.tasks.index(current_task), + ) + .require_schema(TestSteps) + ) + user_instructions: TestSteps = await llm(convo, parser=JSONParser(TestSteps)) + + if len(user_instructions.steps) == 0: + await self.ui.send_message( + "No testing required for this task, moving on to the next one.", source=pythagora_source + ) + log.debug(f"Nothing to do for user testing for task {self.current_state.current_task['description']}") + return None + + user_instructions = json.dumps([test.dict() for test in user_instructions.steps]) + + return user_instructions + + async def _get_route_files(self) -> list[File]: + """Returns the list of file paths that have routes defined in them.""" + + llm = self.get_llm() + convo = AgentConvo(self).template("get_route_files").require_schema(RouteFilePaths) + file_list = await llm(convo, parser=JSONParser(RouteFilePaths)) + route_files: set[str] = set(file_list.files) + + # Sometimes LLM can return a non-existent file, let's make sure to filter those out + return [f for f in self.current_state.files if f.path in route_files] + + async def get_user_feedback( + self, + run_command: str, + user_instructions: str, + last_iteration: Optional[dict], + ) -> tuple[bool, bool, str, str]: + """ + Ask the user to test the app and provide feedback. + + :return (bool, bool, str): Tuple containing "should_iterate", "is_loop" and + "user_feedback" respectively. + + If "should_iterate" is False, the user has confirmed that the app works as expected and there's + nothing for the troubleshooter or problem solver to do. + + If "is_loop" is True, Pythagora is stuck in a loop and needs to consider alternative solutions. + + The last element in the tuple is the user feedback, which may be empty if the user provided no + feedback (eg. if they just clicked on "Continue" or "Start Pair Programming"). + """ + + bug_report = None + change_description = None + hint = None + + is_loop = False + should_iterate = True + extra_info = "restart_app" if not self.current_state.iterations else None + + while True: + await self.ui.send_project_stage({"stage": ProjectStage.GET_USER_FEEDBACK}) + + test_message = "Please check if the app is working" + if user_instructions: + hint = " Here is a description of what should be working:\n\n" + user_instructions + + if run_command: + await self.ui.send_run_command(run_command) + + buttons = { + "continue": "Everything works", + "change": "I want to make a change", + "bug": "There is an issue", + } + + user_response = await self.ask_question( + test_message, + buttons=buttons, + default="continue", + buttons_only=True, + hint=hint, + extra_info=extra_info, + ) + extra_info = None + + if user_response.button == "continue" or user_response.cancelled: + should_iterate = False + break + + elif user_response.button == "change": + await self.ui.send_project_stage({"stage": ProjectStage.DESCRIBE_CHANGE}) + user_description = await self.ask_question( + "Please describe the change you want to make to the project specification (one at a time)", + buttons={"back": "Back"}, + ) + if user_description.button == "back": + continue + change_description = user_description.text + await self.get_relevant_files(user_feedback=change_description) + break + + elif user_response.button == "bug": + await self.ui.send_project_stage({"stage": ProjectStage.DESCRIBE_ISSUE}) + user_description = await self.ask_question( + "Please describe the issue you found (one at a time) and share any relevant server logs", + extra_info="collect_logs", + buttons={"back": "Back"}, + ) + if user_description.button == "back": + continue + bug_report = user_description.text + await self.get_relevant_files(user_feedback=bug_report) + break + + return should_iterate, is_loop, bug_report, change_description + + def try_next_alternative_solution(self, user_feedback: str, user_feedback_qa: list[str]) -> AgentResponse: + """ + Call the ProblemSolver to try an alternative solution. + + Stores the user feedback and sets iteration state so that ProblemSolver will be triggered. + + :param user_feedback: User feedback to store in the iteration state. + :param user_feedback_qa: Additional questions/answers about the problem. + :return: Agent response done. + """ + next_state_iteration = self.next_state.iterations[-1] + next_state_iteration["description"] = "" + next_state_iteration["user_feedback"] = user_feedback + next_state_iteration["user_feedback_qa"] = user_feedback_qa + next_state_iteration["attempts"] += 1 + next_state_iteration["status"] = IterationStatus.PROBLEM_SOLVER + self.next_state.flag_iterations_as_modified() + self.next_state.action = f"Alternative solution (attempt #{next_state_iteration['attempts']})" + return AgentResponse.done(self) + + async def generate_bug_report( + self, + run_command: Optional[str], + user_instructions: str, + user_feedback: str, + ) -> list[str]: + """ + Generate a bug report from the user feedback. + + :param run_command: The command to run to test the app. + :param user_instructions: Instructions on how to test the functionality. + :param user_feedback: The user feedback. + :return: Additional questions and answers to generate a better bug report. + """ + additional_qa = [] + llm = self.get_llm(stream_output=True) + convo = ( + AgentConvo(self) + .template( + "bug_report", + user_instructions=user_instructions, + user_feedback=user_feedback, + # TODO: revisit if we again want to run this in a loop, where this is useful + additional_qa=additional_qa, + ) + .require_schema(BugReportQuestions) + ) + llm_response: BugReportQuestions = await llm(convo, parser=JSONParser(BugReportQuestions)) + + if not llm_response.missing_data: + return [] + + for question in llm_response.missing_data: + if run_command: + await self.ui.send_run_command(run_command) + user_response = await self.ask_question( + question, + buttons={ + "continue": "continue", + "skip": "Skip this question", + "skip-all": "Skip all questions", + }, + allow_empty=False, + ) + if user_response.cancelled or user_response.button == "skip-all": + break + elif user_response.button == "skip": + continue + + additional_qa.append( + { + "question": question, + "answer": user_response.text, + } + ) + + return additional_qa + + async def trace_loop(self, trace_event: str): + state = self.current_state + task_with_loop = { + "task_description": state.current_task["description"], + "task_number": len([t for t in state.tasks if t["status"] == TaskStatus.DONE]) + 1, + "steps": len(state.steps), + "iterations": len(state.iterations), + } + await telemetry.trace_loop(trace_event, task_with_loop) diff --git a/core/cli/__init__.py b/core/cli/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/core/cli/helpers.py b/core/cli/helpers.py new file mode 100644 index 00000000..1eac6d3b --- /dev/null +++ b/core/cli/helpers.py @@ -0,0 +1,332 @@ +import json +import os +import os.path +import sys +from argparse import ArgumentParser, ArgumentTypeError, Namespace +from typing import Optional +from urllib.parse import urlparse +from uuid import UUID + +from core.config import Config, LLMProvider, LocalIPCConfig, ProviderConfig, UIAdapter, get_config, loader +from core.config.env_importer import import_from_dotenv +from core.config.version import get_version +from core.db.session import SessionManager +from core.db.setup import run_migrations +from core.log import setup +from core.state.state_manager import StateManager +from core.ui.base import UIBase +from core.ui.console import PlainConsoleUI +from core.ui.ipc_client import IPCClientUI +from core.ui.virtual import VirtualUI + + +def parse_llm_endpoint(value: str) -> Optional[tuple[LLMProvider, str]]: + """ + Parse --llm-endpoint command-line option. + + Option syntax is: --llm-endpoint : + + :param value: Argument value. + :return: Tuple with LLM provider and URL, or None if the option wasn't provided. + """ + if not value: + return None + + parts = value.split(":", 1) + if len(parts) != 2: + raise ArgumentTypeError("Invalid LLM endpoint format; expected 'provider:url'") + + try: + provider = LLMProvider(parts[0]) + except ValueError as err: + raise ArgumentTypeError(f"Unsupported LLM provider: {err}") + url = urlparse(parts[1]) + if url.scheme not in ("http", "https"): + raise ArgumentTypeError(f"Invalid LLM endpoint URL: {parts[1]}") + + return provider, url.geturl() + + +def parse_llm_key(value: str) -> Optional[tuple[LLMProvider, str]]: + """ + Parse --llm-key command-line option. + + Option syntax is: --llm-key : + + :param value: Argument value. + :return: Tuple with LLM provider and key, or None if if the option wasn't provided. + """ + if not value: + return None + + parts = value.split(":", 1) + if len(parts) != 2: + raise ArgumentTypeError("Invalid LLM endpoint format; expected 'provider:key'") + + try: + provider = LLMProvider(parts[0]) + except ValueError as err: + raise ArgumentTypeError(f"Unsupported LLM provider: {err}") + + return provider, parts[1] + + +def parse_arguments() -> Namespace: + """ + Parse command-line arguments. + + Available arguments: + --help: Show the help message + --config: Path to the configuration file + --show-config: Output the default configuration to stdout + --default-config: Output the configuration to stdout + --level: Log level (debug,info,warning,error,critical) + --database: Database URL + --local-ipc-port: Local IPC port to connect to + --local-ipc-host: Local IPC host to connect to + --version: Show the version and exit + --list: List all projects + --list-json: List all projects in JSON format + --project: Load a specific project + --branch: Load a specific branch + --step: Load a specific step in a project/branch + --llm-endpoint: Use specific API endpoint for the given provider + --llm-key: Use specific LLM key for the given provider + --import-v0: Import data from a v0 (gpt-pilot) database with the given path + --email: User's email address, if provided + --extension-version: Version of the VSCode extension, if used + --no-check: Disable initial LLM API check + --use-git: Use Git for version control + :return: Parsed arguments object. + """ + version = get_version() + + parser = ArgumentParser() + parser.add_argument("--config", help="Path to the configuration file", default="config.json") + parser.add_argument("--show-config", help="Output the default configuration to stdout", action="store_true") + parser.add_argument("--level", help="Log level (debug,info,warning,error,critical)", required=False) + parser.add_argument("--database", help="Database URL", required=False) + parser.add_argument("--local-ipc-port", help="Local IPC port to connect to", type=int, required=False) + parser.add_argument("--local-ipc-host", help="Local IPC host to connect to", default="localhost", required=False) + parser.add_argument("--version", action="version", version=version) + parser.add_argument("--list", help="List all projects", action="store_true") + parser.add_argument("--list-json", help="List all projects in JSON format", action="store_true") + parser.add_argument("--project", help="Load a specific project", type=UUID, required=False) + parser.add_argument("--branch", help="Load a specific branch", type=UUID, required=False) + parser.add_argument("--step", help="Load a specific step in a project/branch", type=int, required=False) + parser.add_argument("--delete", help="Delete a specific project", type=UUID, required=False) + parser.add_argument( + "--llm-endpoint", + help="Use specific API endpoint for the given provider", + type=parse_llm_endpoint, + action="append", + required=False, + ) + parser.add_argument( + "--llm-key", + help="Use specific LLM key for the given provider", + type=parse_llm_key, + action="append", + required=False, + ) + parser.add_argument( + "--import-v0", + help="Import data from a v0 (gpt-pilot) database with the given path", + required=False, + ) + parser.add_argument("--email", help="User's email address", required=False) + parser.add_argument("--extension-version", help="Version of the VSCode extension", required=False) + parser.add_argument("--no-check", help="Disable initial LLM API check", action="store_true") + parser.add_argument("--use-git", help="Use Git for version control", action="store_true", required=False) + return parser.parse_args() + + +def load_config(args: Namespace) -> Optional[Config]: + """ + Load Pythagora JSON configuration file and apply command-line arguments. + + :param args: Command-line arguments (at least `config` must be present). + :return: Configuration object, or None if config couldn't be loaded. + """ + if not os.path.isfile(args.config): + imported = import_from_dotenv(args.config) + if not imported: + print(f"Configuration file not found: {args.config}; using default", file=sys.stderr) + return get_config() + + try: + config = loader.load(args.config) + except ValueError as err: + print(f"Error parsing config file {args.config}: {err}", file=sys.stderr) + return None + + if args.level: + config.log.level = args.level.upper() + + if args.database: + config.db.url = args.database + + if args.local_ipc_port: + config.ui = LocalIPCConfig(port=args.local_ipc_port, host=args.local_ipc_host) + + if args.llm_endpoint: + for provider, endpoint in args.llm_endpoint: + if provider not in config.llm: + config.llm[provider] = ProviderConfig() + config.llm[provider].base_url = endpoint + + if args.llm_key: + for provider, key in args.llm_key: + if provider not in config.llm: + config.llm[provider] = ProviderConfig() + config.llm[provider].api_key = key + + try: + Config.model_validate(config) + except ValueError as err: + print(f"Configuration error: {err}", file=sys.stderr) + return None + + return config + + +async def list_projects_json(db: SessionManager): + """ + List all projects in the database in JSON format. + """ + sm = StateManager(db) + projects = await sm.list_projects() + + data = [] + for project in projects: + last_updated = None + p = { + "name": project.name, + "id": project.id.hex, + "branches": [], + } + for branch in project.branches: + b = { + "name": branch.name, + "id": branch.id.hex, + "steps": [], + } + for state in branch.states: + if not last_updated or state.created_at > last_updated: + last_updated = state.created_at + s = { + "name": state.action or f"Step #{state.step_index}", + "step": state.step_index, + } + b["steps"].append(s) + if b["steps"]: + b["steps"][-1]["name"] = "Latest step" + p["branches"].append(b) + p["updated_at"] = last_updated.isoformat() if last_updated else None + data.append(p) + + print(json.dumps(data, indent=2)) + + +async def list_projects(db: SessionManager): + """ + List all projects in the database. + """ + sm = StateManager(db) + projects = await sm.list_projects() + + print(f"Available projects ({len(projects)}):") + for project in projects: + print(f"* {project.name} ({project.id})") + for branch in project.branches: + last_step = max(state.step_index for state in branch.states) + print(f" - {branch.name} ({branch.id}) - last step: {last_step}") + + +async def load_project( + sm: StateManager, + project_id: Optional[UUID] = None, + branch_id: Optional[UUID] = None, + step_index: Optional[int] = None, +) -> bool: + """ + Load a project from the database. + + :param sm: State manager. + :param project_id: Project ID (optional, loads the last step in the main branch). + :param branch_id: Branch ID (optional, loads the last step in the branch). + :param step_index: Step index (optional, loads the state at the given step). + :return: True if the project was loaded successfully, False otherwise. + """ + step_txt = f" step {step_index}" if step_index else "" + + if branch_id: + project_state = await sm.load_project(branch_id=branch_id, step_index=step_index) + if project_state: + return True + else: + print(f"Branch {branch_id}{step_txt} not found; use --list to list all projects", file=sys.stderr) + return False + + elif project_id: + project_state = await sm.load_project(project_id=project_id, step_index=step_index) + if project_state: + return True + else: + print(f"Project {project_id}{step_txt} not found; use --list to list all projects", file=sys.stderr) + return False + + return False + + +async def delete_project(db: SessionManager, project_id: UUID) -> bool: + """ + Delete a project from a database. + + :param sm: State manager. + :param project_id: Project ID. + :return: True if project was deleted, False otherwise. + """ + + sm = StateManager(db) + return await sm.delete_project(project_id) + + +def show_config(): + """ + Print the current configuration to stdout. + """ + cfg = get_config() + print(cfg.model_dump_json(indent=2)) + + +def init() -> tuple[UIBase, SessionManager, Namespace]: + """ + Initialize the application. + + Loads configuration, sets up logging and UI, initializes the database + and runs database migrations. + + :return: Tuple with UI, db session manager, file manager, and command-line arguments. + """ + args = parse_arguments() + config = load_config(args) + if not config: + return (None, None, args) + + setup(config.log, force=True) + + if config.ui.type == UIAdapter.IPC_CLIENT: + ui = IPCClientUI(config.ui) + elif config.ui.type == UIAdapter.VIRTUAL: + ui = VirtualUI(config.ui.inputs) + else: + ui = PlainConsoleUI() + + run_migrations(config.db) + db = SessionManager(config.db) + + return (ui, db, args) + + +__all__ = ["parse_arguments", "load_config", "list_projects_json", "list_projects", "load_project", "init"] diff --git a/core/cli/main.py b/core/cli/main.py new file mode 100644 index 00000000..3ae9b224 --- /dev/null +++ b/core/cli/main.py @@ -0,0 +1,320 @@ +import asyncio +import atexit +import signal +import sys +from argparse import Namespace +from asyncio import run + +from core.agents.orchestrator import Orchestrator +from core.cli.helpers import delete_project, init, list_projects, list_projects_json, load_project, show_config +from core.config import LLMProvider, get_config +from core.db.session import SessionManager +from core.db.v0importer import LegacyDatabaseImporter +from core.llm.anthropic_client import CustomAssertionError +from core.llm.base import APIError, BaseLLMClient +from core.log import get_logger +from core.state.state_manager import StateManager +from core.telemetry import telemetry +from core.ui.base import ProjectStage, UIBase, UIClosedError, UserInput, pythagora_source + +log = get_logger(__name__) + + +telemetry_sent = False + + +async def cleanup(ui: UIBase): + global telemetry_sent + if not telemetry_sent: + await telemetry.send() + telemetry_sent = True + await ui.stop() + + +def sync_cleanup(ui: UIBase): + asyncio.run(cleanup(ui)) + + +async def run_project(sm: StateManager, ui: UIBase, args) -> bool: + """ + Work on the project. + + Starts the orchestrator agent with the newly loaded/created project + and runs it until the orchestrator decides to exit. + + :param sm: State manager. + :param ui: User interface. + :param args: Command-line arguments. + :return: True if the orchestrator exited successfully, False otherwise. + """ + + telemetry.set("app_id", str(sm.project.id)) + telemetry.set("initial_prompt", sm.current_state.specification.description) + + orca = Orchestrator(sm, ui, args=args) + success = False + try: + success = await orca.run() + telemetry.set("end_result", "success:exit" if success else "failure:api-error") + except (KeyboardInterrupt, UIClosedError): + log.info("Interrupted by user") + telemetry.set("end_result", "interrupt") + await sm.rollback() + except APIError as err: + log.warning(f"LLM API error occurred: {err.message}") + await ui.send_message( + f"Stopping Pythagora due to an error while calling the LLM API: {err.message}", + source=pythagora_source, + ) + telemetry.set("end_result", "failure:api-error") + await sm.rollback() + except CustomAssertionError as err: + log.warning(f"Anthropic assertion error occurred: {str(err)}") + await ui.send_message( + f"Stopping Pythagora due to an error inside Anthropic SDK. {str(err)}", + source=pythagora_source, + ) + telemetry.set("end_result", "failure:assertion-error") + await sm.rollback() + except Exception as err: + log.error(f"Uncaught exception: {err}", exc_info=True) + stack_trace = telemetry.record_crash(err) + await sm.rollback() + await ui.send_message( + f"Stopping Pythagora due to error:\n\n{stack_trace}", + source=pythagora_source, + ) + + return success + + +async def llm_api_check(ui: UIBase) -> bool: + """ + Check whether the configured LLMs are reachable in parallel. + + :param ui: UI we'll use to report any issues + :return: True if all the LLMs are reachable. + """ + + config = get_config() + + async def handler(*args, **kwargs): + pass + + checked_llms: set[LLMProvider] = set() + tasks = [] + + async def check_llm(llm_config): + if llm_config.provider + llm_config.model in checked_llms: + return True + + checked_llms.add(llm_config.provider + llm_config.model) + client_class = BaseLLMClient.for_provider(llm_config.provider) + llm_client = client_class(llm_config, stream_handler=handler, error_handler=handler) + try: + resp = await llm_client.api_check() + if not resp: + await ui.send_message( + f"API check for {llm_config.provider.value} {llm_config.model} failed.", + source=pythagora_source, + ) + log.warning(f"API check for {llm_config.provider.value} {llm_config.model} failed.") + return False + else: + log.info(f"API check for {llm_config.provider.value} {llm_config.model} succeeded.") + return True + except APIError as err: + await ui.send_message( + f"API check for {llm_config.provider.value} {llm_config.model} failed with: {err}", + source=pythagora_source, + ) + log.warning(f"API check for {llm_config.provider.value} failed with: {err}") + return False + + for llm_config in config.all_llms(): + tasks.append(check_llm(llm_config)) + + results = await asyncio.gather(*tasks) + + success = all(results) + + if not success: + telemetry.set("end_result", "failure:api-error") + + return success + + +async def start_new_project(sm: StateManager, ui: UIBase) -> bool: + """ + Start a new project. + + :param sm: State manager. + :param ui: User interface. + :return: True if the project was created successfully, False otherwise. + """ + + stack = await ui.ask_question( + "What do you want to use to build your app?", + allow_empty=False, + buttons={"node": "Node.js", "other": "Other (coming soon)"}, + buttons_only=True, + source=pythagora_source, + full_screen=True, + ) + + if stack.button == "other": + language = await ui.ask_question( + "What language you want to use?", + allow_empty=False, + source=pythagora_source, + full_screen=True, + ) + await telemetry.trace_code_event( + "stack-choice-other", + {"language": language.text}, + ) + await ui.send_message("Thank you for submitting your request to support other languages.") + return False + elif stack.button == "node": + await telemetry.trace_code_event( + "stack-choice", + {"language": "node"}, + ) + elif stack.button == "python": + await telemetry.trace_code_event( + "stack-choice", + {"language": "python"}, + ) + + while True: + try: + await ui.send_project_stage({"stage": ProjectStage.PROJECT_NAME}) + user_input = await ui.ask_question( + "What is the project name?", + allow_empty=False, + source=pythagora_source, + full_screen=True, + ) + except (KeyboardInterrupt, UIClosedError): + user_input = UserInput(cancelled=True) + + if user_input.cancelled: + return False + + project_name = user_input.text.strip() + if not project_name: + await ui.send_message("Please choose a project name", source=pythagora_source) + elif len(project_name) > 100: + await ui.send_message("Please choose a shorter project name", source=pythagora_source) + else: + break + + project_state = await sm.create_project(project_name) + return project_state is not None + + +async def run_pythagora_session(sm: StateManager, ui: UIBase, args: Namespace): + """ + Run a Pythagora session. + + :param sm: State manager. + :param ui: User interface. + :param args: Command-line arguments. + :return: True if the application ran successfully, False otherwise. + """ + + if not args.no_check: + if not await llm_api_check(ui): + await ui.send_message( + "Pythagora cannot start because the LLM API is not reachable.", + source=pythagora_source, + ) + return False + + if args.project or args.branch or args.step: + telemetry.set("is_continuation", True) + success = await load_project(sm, args.project, args.branch, args.step) + if not success: + return False + else: + success = await start_new_project(sm, ui) + if not success: + return False + + return await run_project(sm, ui, args) + + +async def async_main( + ui: UIBase, + db: SessionManager, + args: Namespace, +) -> bool: + """ + Main application coroutine. + + :param ui: User interface. + :param db: Database session manager. + :param args: Command-line arguments. + :return: True if the application ran successfully, False otherwise. + """ + global telemetry_sent + + if args.list: + await list_projects(db) + return True + elif args.list_json: + await list_projects_json(db) + return True + if args.show_config: + show_config() + return True + elif args.import_v0: + importer = LegacyDatabaseImporter(db, args.import_v0) + await importer.import_database() + return True + elif args.delete: + success = await delete_project(db, args.delete) + return success + + telemetry.set("user_contact", args.email) + if args.extension_version: + telemetry.set("is_extension", True) + telemetry.set("extension_version", args.extension_version) + + sm = StateManager(db, ui) + ui_started = await ui.start() + if not ui_started: + return False + + telemetry.start() + + # Set up signal handlers + def signal_handler(sig, frame): + if not telemetry_sent: + sync_cleanup(ui) + sys.exit(0) + + for sig in (signal.SIGINT, signal.SIGTERM): + signal.signal(sig, signal_handler) + + # Register the cleanup function + atexit.register(sync_cleanup, ui) + + try: + success = await run_pythagora_session(sm, ui, args) + finally: + await cleanup(ui) + + return success + + +def run_pythagora(): + ui, db, args = init() + if not ui or not db: + return -1 + success = run(async_main(ui, db, args)) + return 0 if success else -1 + + +if __name__ == "__main__": + sys.exit(run_pythagora()) diff --git a/core/config/__init__.py b/core/config/__init__.py new file mode 100644 index 00000000..4e142368 --- /dev/null +++ b/core/config/__init__.py @@ -0,0 +1,522 @@ +from enum import Enum +from os.path import abspath, dirname, isdir, join +from typing import Any, Literal, Optional, Union + +from pydantic import BaseModel, ConfigDict, Field, field_validator +from typing_extensions import Annotated + +ROOT_DIR = abspath(join(dirname(__file__), "..", "..")) +DEFAULT_IGNORE_PATHS = [ + ".git", + ".gpt-pilot", + ".idea", + ".vscode", + ".next", + ".DS_Store", + "__pycache__", + "site-packages", + "node_modules", + "package-lock.json", + "venv", + ".venv", + "dist", + "build", + "target", + "*.min.js", + "*.min.css", + "*.svg", + "*.csv", + "*.log", + "go.sum", + "migration_lock.toml", +] +IGNORE_SIZE_THRESHOLD = 50000 # 50K+ files are ignored by default + +# Agents with sane setup in the default configuration +DEFAULT_AGENT_NAME = "default" +CODE_MONKEY_AGENT_NAME = "CodeMonkey" +CODE_REVIEW_AGENT_NAME = "CodeMonkey.code_review" +DESCRIBE_FILES_AGENT_NAME = "CodeMonkey.describe_files" +CHECK_LOGS_AGENT_NAME = "BugHunter.check_logs" +PARSE_TASK_AGENT_NAME = "Developer.parse_task" +TASK_BREAKDOWN_AGENT_NAME = "Developer.breakdown_current_task" +TROUBLESHOOTER_BUG_REPORT = "Troubleshooter.generate_bug_report" +TROUBLESHOOTER_GET_RUN_COMMAND = "Troubleshooter.get_run_command" +TECH_LEAD_PLANNING = "TechLead.plan_epic" +TECH_LEAD_EPIC_BREAKDOWN = "TechLead.epic_breakdown" +SPEC_WRITER_AGENT_NAME = "SpecWriter" +GET_RELEVANT_FILES_AGENT_NAME = "get_relevant_files" +FRONTEND_AGENT_NAME = "Frontend" + +# Endpoint for the external documentation +EXTERNAL_DOCUMENTATION_API = "http://docs-pythagora-io-439719575.us-east-1.elb.amazonaws.com" + + +class _StrictModel(BaseModel): + """ + Pydantic parser configuration options. + """ + + model_config = ConfigDict( + extra="forbid", + ) + + +class LLMProvider(str, Enum): + """ + Supported LLM providers. + """ + + OPENAI = "openai" + ANTHROPIC = "anthropic" + GROQ = "groq" + LM_STUDIO = "lm-studio" + AZURE = "azure" + + +class UIAdapter(str, Enum): + """ + Supported UI adapters. + """ + + PLAIN = "plain" + IPC_CLIENT = "ipc-client" + VIRTUAL = "virtual" + + +class ProviderConfig(_StrictModel): + """ + LLM provider configuration. + """ + + base_url: Optional[str] = Field( + None, + description="Base URL for the provider's API (if different from the provider default)", + ) + api_key: Optional[str] = Field( + None, + description="API key to use for authentication (if not set, provider uses default from environment variable)", + ) + connect_timeout: float = Field( + default=60.0, + description="Timeout (in seconds) for connecting to the provider's API", + ge=0.0, + ) + read_timeout: float = Field( + default=60.0, + description="Timeout (in seconds) for receiving a new chunk of data from the response stream", + ge=0.0, + ) + extra: Optional[dict[str, Any]] = Field( + None, + description="Extra provider-specific configuration", + ) + + +class AgentLLMConfig(_StrictModel): + """ + Configuration for the various LLMs used by Pythagora. + + Each Agent has an LLM provider, from the Enum LLMProvider. If + AgentLLMConfig is not specified, default will be used. + """ + + provider: Optional[LLMProvider] = Field(default=LLMProvider.OPENAI, description="LLM provider") + model: str = Field(description="Model to use", default="gpt-4o-2024-05-13") + temperature: float = Field( + default=0.5, + description="Temperature to use for sampling", + ge=0.0, + le=1.0, + ) + + +class LLMConfig(_StrictModel): + """ + Complete agent-specific configuration for an LLM. + """ + + provider: LLMProvider = LLMProvider.OPENAI + model: str = Field(description="Model to use") + base_url: Optional[str] = Field( + None, + description="Base URL for the provider's API (if different from the provider default)", + ) + api_key: Optional[str] = Field( + None, + description="API key to use for authentication (if not set, provider uses default from environment variable)", + ) + temperature: float = Field( + default=0.5, + description="Temperature to use for sampling", + ge=0.0, + le=1.0, + ) + connect_timeout: float = Field( + default=60.0, + description="Timeout (in seconds) for connecting to the provider's API", + ge=0.0, + ) + read_timeout: float = Field( + default=60.0, + description="Timeout (in seconds) for receiving a new chunk of data from the response stream", + ge=0.0, + ) + extra: Optional[dict[str, Any]] = Field( + None, + description="Extra provider-specific configuration", + ) + + @classmethod + def from_provider_and_agent_configs(cls, provider: ProviderConfig, agent: AgentLLMConfig): + return cls( + provider=agent.provider, + model=agent.model, + base_url=provider.base_url, + api_key=provider.api_key, + temperature=agent.temperature, + connect_timeout=provider.connect_timeout, + read_timeout=provider.read_timeout, + extra=provider.extra, + ) + + +class PromptConfig(_StrictModel): + """ + Configuration for prompt templates: + """ + + paths: list[str] = Field( + [join(ROOT_DIR, "core", "prompts")], + description="List of directories to search for prompt templates", + ) + + @field_validator("paths") + @classmethod + def validate_paths(cls, v: list[str]) -> list[str]: + for path in v: + if not isdir(path): + raise ValueError(f"Invalid prompt path: {path}") + return v + + +class LogConfig(_StrictModel): + """ + Configuration for logging. + """ + + level: str = Field( + "DEBUG", + description="Logging level", + pattern=r"^(DEBUG|INFO|WARNING|ERROR|CRITICAL)$", + ) + format: str = Field( + "%(asctime)s %(levelname)s [%(name)s] %(message)s", + description="Logging format", + ) + output: Optional[str] = Field( + "pythagora.log", + description="Output file for logs (if not specified, logs are printed to stderr)", + ) + + +class DBConfig(_StrictModel): + """ + Configuration for database connections. + + Supported URL schemes: + + * sqlite+aiosqlite: SQLite database using the aiosqlite driver + """ + + url: str = Field( + "sqlite+aiosqlite:///data/database/pythagora.db", + description="Database connection URL", + ) + debug_sql: bool = Field(False, description="Log all SQL queries to the console") + + @field_validator("url") + @classmethod + def validate_url_scheme(cls, v: str) -> str: + if v.startswith("sqlite+aiosqlite://"): + return v + if v.startswith("postgresql+asyncpg://"): + try: + import asyncpg # noqa: F401 + except ImportError: + raise ValueError("To use PostgreSQL database, please install `asyncpg` and `psycopg2` packages") + return v + raise ValueError(f"Unsupported database URL scheme in: {v}") + + +class PlainUIConfig(_StrictModel): + """ + Configuration for plaintext console UI. + """ + + type: Literal[UIAdapter.PLAIN] = UIAdapter.PLAIN + + +class LocalIPCConfig(_StrictModel): + """ + Configuration for VSCode extension IPC client. + """ + + type: Literal[UIAdapter.IPC_CLIENT] = UIAdapter.IPC_CLIENT + host: str = "localhost" + port: int = 8125 + + +class VirtualUIConfig(_StrictModel): + """ + Configuration for the virtual UI. + """ + + type: Literal[UIAdapter.VIRTUAL] = UIAdapter.VIRTUAL + inputs: list[Any] + + +UIConfig = Annotated[ + Union[PlainUIConfig, LocalIPCConfig, VirtualUIConfig], + Field(discriminator="type"), +] + + +class FileSystemType(str, Enum): + """ + Supported filesystem types. + """ + + MEMORY = "memory" + LOCAL = "local" + + +class FileSystemConfig(_StrictModel): + """ + Configuration for project workspace. + """ + + type: Literal[FileSystemType.LOCAL] = FileSystemType.LOCAL + workspace_root: str = Field( + join(ROOT_DIR, "workspace"), + description="Workspace directory containing all the projects", + ) + ignore_paths: list[str] = Field( + DEFAULT_IGNORE_PATHS, + description="List of paths to ignore when scanning for files and folders", + ) + ignore_size_threshold: int = Field( + IGNORE_SIZE_THRESHOLD, + description="Files larger than this size should be ignored", + ) + + +class Config(_StrictModel): + """ + Pythagora Core configuration + """ + + llm: dict[LLMProvider, ProviderConfig] = Field( + default={ + LLMProvider.OPENAI: ProviderConfig(), + LLMProvider.ANTHROPIC: ProviderConfig(), + } + ) + agent: dict[str, AgentLLMConfig] = Field( + default={ + DEFAULT_AGENT_NAME: AgentLLMConfig(), + CHECK_LOGS_AGENT_NAME: AgentLLMConfig( + provider=LLMProvider.ANTHROPIC, + model="claude-3-5-sonnet-20241022", + temperature=0.5, + ), + CODE_MONKEY_AGENT_NAME: AgentLLMConfig( + provider=LLMProvider.ANTHROPIC, + model="claude-3-5-sonnet-20241022", + temperature=0.0, + ), + CODE_REVIEW_AGENT_NAME: AgentLLMConfig( + provider=LLMProvider.ANTHROPIC, + model="claude-3-5-sonnet-20240620", + temperature=0.0, + ), + DESCRIBE_FILES_AGENT_NAME: AgentLLMConfig( + provider=LLMProvider.OPENAI, + model="gpt-4o-mini-2024-07-18", + temperature=0.0, + ), + FRONTEND_AGENT_NAME: AgentLLMConfig( + provider=LLMProvider.ANTHROPIC, + model="claude-3-5-sonnet-20241022", + temperature=0.0, + ), + GET_RELEVANT_FILES_AGENT_NAME: AgentLLMConfig( + provider=LLMProvider.OPENAI, + model="gpt-4o-2024-05-13", + temperature=0.5, + ), + PARSE_TASK_AGENT_NAME: AgentLLMConfig( + provider=LLMProvider.ANTHROPIC, + model="claude-3-5-sonnet-20241022", + temperature=0.0, + ), + SPEC_WRITER_AGENT_NAME: AgentLLMConfig( + provider=LLMProvider.OPENAI, + model="gpt-4-0125-preview", + temperature=0.0, + ), + TASK_BREAKDOWN_AGENT_NAME: AgentLLMConfig( + provider=LLMProvider.ANTHROPIC, + model="claude-3-5-sonnet-20241022", + temperature=0.5, + ), + TECH_LEAD_PLANNING: AgentLLMConfig( + provider=LLMProvider.ANTHROPIC, + model="claude-3-5-sonnet-20240620", + temperature=0.5, + ), + TECH_LEAD_EPIC_BREAKDOWN: AgentLLMConfig( + provider=LLMProvider.ANTHROPIC, + model="claude-3-5-sonnet-20241022", + temperature=0.5, + ), + TROUBLESHOOTER_BUG_REPORT: AgentLLMConfig( + provider=LLMProvider.ANTHROPIC, + model="claude-3-5-sonnet-20240620", + temperature=0.5, + ), + TROUBLESHOOTER_GET_RUN_COMMAND: AgentLLMConfig( + provider=LLMProvider.ANTHROPIC, + model="claude-3-5-sonnet-20240620", + temperature=0.0, + ), + } + ) + prompt: PromptConfig = PromptConfig() + log: LogConfig = LogConfig() + db: DBConfig = DBConfig() + ui: UIConfig = PlainUIConfig() + fs: FileSystemConfig = FileSystemConfig() + + def llm_for_agent(self, agent_name: str = "default") -> LLMConfig: + """ + Fetch an LLM configuration for a given agent. + + If the agent specific configuration doesn't exist, returns the configuration + for the 'default' agent. + """ + + agent_name = agent_name if agent_name in self.agent else "default" + agent_config = self.agent[agent_name] + provider_config = self.llm[agent_config.provider] + return LLMConfig.from_provider_and_agent_configs(provider_config, agent_config) + + def all_llms(self) -> list[LLMConfig]: + """ + Get configuration for all defined LLMs. + """ + + return [self.llm_for_agent(agent) for agent in self.agent] + + +class ConfigLoader: + """ + Configuration loader takes care of loading and parsing configuration files. + + The default loader is already initialized as `core.config.loader`. To + load the configuration from a file, use `core.config.loader.load(path)`. + + To get the current configuration, use `core.config.get_config()`. + """ + + config: Config + config_path: Optional[str] + + def __init__(self): + self.config_path = None + self.config = Config() + + @staticmethod + def _remove_json_comments(json_str: str) -> str: + """ + Remove comments from a JSON string. + + Removes all lines that start with "//" from the JSON string. + + :param json_str: JSON string with comments. + :return: JSON string without comments. + """ + return "\n".join([line for line in json_str.splitlines() if not line.strip().startswith("//")]) + + @classmethod + def from_json(cls: "ConfigLoader", config: str) -> Config: + """ + Parse JSON Into a Config object. + + :param config: JSON string to parse. + :return: Config object. + """ + return Config.model_validate_json(cls._remove_json_comments(config), strict=True) + + def load(self, path: str) -> Config: + """ + Load a configuration from a file. + + :param path: Path to the configuration file. + :return: Config object. + """ + with open(path, "rb") as f: + raw_config = f.read() + + if b"\x00" in raw_config: + encoding = "utf-16" + else: + encoding = "utf-8" + + text_config = raw_config.decode(encoding) + self.config = self.from_json(text_config) + self.config_path = path + return self.config + + +loader = ConfigLoader() + + +def adapt_for_bedrock(config: Config) -> Config: + """ + Adapt the configuration for use with Bedrock. + + :param config: Configuration to adapt. + :return: Adapted configuration. + """ + if "anthropic" not in config.llm: + return config + + if config.llm["anthropic"].base_url is None or "bedrock/anthropic" not in config.llm["anthropic"].base_url: + return config + + replacement_map = { + "claude-3-5-sonnet-20241022": "us.anthropic.claude-3-5-sonnet-20241022-v2:0", + "claude-3-5-sonnet-20240620": "us.anthropic.claude-3-5-sonnet-20240620-v1:0", + "claude-3-sonnet-20240229": "us.anthropic.claude-3-sonnet-20240229-v1:0", + "claude-3-haiku-20240307": "us.anthropic.claude-3-haiku-20240307-v1:0", + "claude-3-opus-20240229": "us.anthropic.claude-3-opus-20240229-v1:0", + } + + for agent in config.agent: + if config.agent[agent].model in replacement_map: + config.agent[agent].model = replacement_map[config.agent[agent].model] + + return config + + +def get_config() -> Config: + """ + Return current configuration. + + :return: Current configuration object. + """ + return adapt_for_bedrock(loader.config) + + +__all__ = ["loader", "get_config"] diff --git a/core/config/env_importer.py b/core/config/env_importer.py new file mode 100644 index 00000000..f698240b --- /dev/null +++ b/core/config/env_importer.py @@ -0,0 +1,91 @@ +from os.path import dirname, exists, join + +from dotenv import dotenv_values + +from core.config import Config, LLMProvider, ProviderConfig, loader + + +def import_from_dotenv(new_config_path: str) -> bool: + """ + Import configuration from old gpt-pilot .env file and save it to a new format. + + If the configuration is already loaded, does nothing. If the target file + already exists, it's parsed as is (it's not overwritten). + + Otherwise, loads the values from `pilot/.env` file and creates a new configuration + with the relevant settings. + + This intentionally DOES NOT load the .env variables into the current process + environments, to avoid polluting it with old settings. + + :param new_config_path: Path to save the new configuration file. + :return: True if the configuration was imported, False otherwise. + """ + if loader.config_path or exists(new_config_path): + # Config already exists, nothing to do + return True + + env_path = join(dirname(__file__), "..", "..", "pilot", ".env") + if not exists(env_path): + return False + + values = dotenv_values(env_path) + if not values: + return False + + config = convert_config(values) + + with open(new_config_path, "w", encoding="utf-8") as fp: + fp.write(config.model_dump_json(indent=2)) + + return True + + +def convert_config(values: dict) -> Config: + config = Config() + + for provider in LLMProvider: + endpoint = values.get(f"{provider.value.upper()}_ENDPOINT") + key = values.get(f"{provider.value.upper()}_API_KEY") + + if provider == LLMProvider.OPENAI: + # OpenAI is also used for Azure and OpenRouter and local LLMs + if endpoint is None: + endpoint = values.get("AZURE_ENDPOINT") + if endpoint is None: + endpoint = values.get("OPENROUTER_ENDPOINT") + + if key is None: + key = values.get("AZURE_API_KEY") + if key is None: + key = values.get("OPENROUTER_API_KEY") + if key and endpoint is None: + endpoint = "https://openrouter.ai/api/v1/chat/completions" + + if endpoint or key and provider not in config.llm: + config.llm[provider] = ProviderConfig() + + if endpoint: + endpoint = endpoint.replace("chat/completions", "") + config.llm[provider].base_url = endpoint + if key: + config.llm[provider].api_key = key + + model = values.get("MODEL_NAME") + if model: + provider = "openai" + if "/" in model: + provider, model = model.split("/", 1) + + try: + agent_provider = LLMProvider(provider.upper()) + except ValueError: + agent_provider = LLMProvider.OPENAI + + config.agent["default"].model = model + config.agent["default"].provider = agent_provider + + ignore_paths = [p for p in values.get("IGNORE_PATHS", "").split(",") if p] + if ignore_paths: + config.fs.ignore_paths += ignore_paths + return config diff --git a/core/config/magic_words.py b/core/config/magic_words.py new file mode 100644 index 00000000..e27a272c --- /dev/null +++ b/core/config/magic_words.py @@ -0,0 +1,59 @@ +PROBLEM_IDENTIFIED = "PROBLEM_IDENTIFIED" +ADD_LOGS = "ADD_LOGS" +THINKING_LOGS = [ + "Pythagora is crunching the numbers...", + "Pythagora is deep in thought...", + "Pythagora is analyzing your request...", + "Pythagora is brewing up a solution...", + "Pythagora is putting the pieces together...", + "Pythagora is working its magic...", + "Pythagora is crafting the perfect response...", + "Pythagora is decoding your query...", + "Pythagora is on the case...", + "Pythagora is computing an answer...", + "Pythagora is sorting through the data...", + "Pythagora is gathering insights...", + "Pythagora is making connections...", + "Pythagora is tuning the algorithms...", + "Pythagora is piecing together the puzzle...", + "Pythagora is scanning the possibilities...", + "Pythagora is engineering a response...", + "Pythagora is building the answer...", + "Pythagora is mapping out a solution...", + "Pythagora is figuring this out for you...", + "Pythagora is thinking hard right now...", + "Pythagora is working for you, so relax!", + "Pythagora might take some time to figure this out...", +] +GITIGNORE_CONTENT = """# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +lerna-debug.log* + +node_modules +dist +dist-ssr +*.local + +# Editor directories and files +.vscode/* +!.vscode/extensions.json +.idea +.DS_Store +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? + +# SQLite databases, data files +*.db +*.csv + +# Keep environment variables out of version control +.env +""" diff --git a/core/config/user_settings.py b/core/config/user_settings.py new file mode 100644 index 00000000..fb53672b --- /dev/null +++ b/core/config/user_settings.py @@ -0,0 +1,94 @@ +import sys +from os import getenv, makedirs +from pathlib import Path +from uuid import uuid4 + +from pydantic import BaseModel, Field, PrivateAttr + +from core.log import get_logger + +log = get_logger(__name__) + + +SETTINGS_APP_NAME = "GPT Pilot" +DEFAULT_TELEMETRY_ENDPOINT = "https://api.pythagora.io/telemetry" + + +class TelemetrySettings(BaseModel): + id: str = Field(default_factory=lambda: uuid4().hex, description="Unique telemetry ID") + enabled: bool = Field(True, description="Whether telemetry should send stats to the server") + endpoint: str = Field(DEFAULT_TELEMETRY_ENDPOINT, description="Telemetry server endpoint") + + +def resolve_config_dir() -> Path: + """ + Figure out where to store the global config file(s). + + :return: path to the desired location config directory + + See the UserSettings docstring for details on how the config directory is + determined. + """ + posix_app_name = SETTINGS_APP_NAME.replace(" ", "-").lower() + + xdg_config_home = getenv("XDG_CONFIG_HOME") + if xdg_config_home: + return Path(xdg_config_home) / Path(posix_app_name) + + if sys.platform == "win32" and getenv("APPDATA"): + return Path(getenv("APPDATA")) / Path(SETTINGS_APP_NAME) + + return Path("~").expanduser() / Path(f".{posix_app_name}") + + +class UserSettings(BaseModel): + """ + This object holds all the global user settings, that are applicable for + all Pythagora/GPT-Pilot installations. + + The use settings are stored in a JSON file in the config directory. + + The config directory is determined by the following rules: + * If the XDG_CONFIG_HOME environment variable is set (desktop Linux), use that. + * If the APPDATA environment variable is set (Windows), use that. + * Otherwise, use the POSIX default ~/. (MacOS, server Linux). + + This is a singleton object, use it by importing the instance directly + from the module: + + >>> from config.user_settings import settings + >>> print(settings.telemetry.id) + >>> print(settings.config_path) + """ + + telemetry: TelemetrySettings = TelemetrySettings() + _config_path: str = PrivateAttr("") + + @staticmethod + def load(): + config_path = resolve_config_dir() / "config.json" + + if not config_path.exists(): + default = UserSettings() + default._config_path = config_path + default.save() + + with open(config_path, "r", encoding="utf-8") as fp: + settings = UserSettings.model_validate_json(fp.read()) + settings._config_path = str(config_path) + return settings + + def save(self): + makedirs(Path(self._config_path).parent, exist_ok=True) + with open(self._config_path, "w", encoding="utf-8") as fp: + fp.write(self.model_dump_json(indent=2)) + + @property + def config_path(self): + return self._config_path + + +settings = UserSettings.load() + + +__all__ = ["settings"] diff --git a/core/config/version.py b/core/config/version.py new file mode 100644 index 00000000..ee9feb3a --- /dev/null +++ b/core/config/version.py @@ -0,0 +1,86 @@ +import re +from os.path import abspath, basename, dirname, isdir, isfile, join +from typing import Optional + +GIT_DIR_PATH = abspath(join(dirname(__file__), "..", "..", ".git")) + + +def get_git_commit() -> Optional[str]: + """ + Return the current git commit (if running from a repo). + + :return: commit hash or None if not running from a git repo + """ + + if not isdir(GIT_DIR_PATH): + return None + + git_head = join(GIT_DIR_PATH, "HEAD") + if not isfile(git_head): + return None + + with open(git_head, "r", encoding="utf-8") as f: + ref = f.read().strip() + + # Direct reference to commit hash + if not ref.startswith("ref: "): + return ref + + # Follow the reference + ref = ref[5:] + ref_path = join(GIT_DIR_PATH, ref) + + # Dangling reference, return the reference name + if not isfile(ref_path): + return basename(ref_path) + + # Return the reference commit hash + with open(ref_path, "r", encoding="utf-8") as f: + return f.read().strip() + + +def get_package_version() -> str: + """ + Get package version as defined pyproject.toml. + + If not found, returns "0.0.0." + + :return: package version as defined in pyproject.toml + """ + UNKNOWN = "0.0.0" + PYPOETRY_VERSION_PATTERN = re.compile(r'^\s*version\s*=\s*"(.*)"\s*(#.*)?$') + + pyproject_path = join(dirname(__file__), "..", "..", "pyproject.toml") + if not isfile(pyproject_path): + return UNKNOWN + + with open(pyproject_path, "r", encoding="utf-8") as fp: + for line in fp: + m = PYPOETRY_VERSION_PATTERN.match(line) + if m: + return m.group(1) + + return UNKNOWN + + +def get_version() -> str: + """ + Find and return the current version of Pythagora Core. + + The version string is built from the package version and the current + git commit hash (if running from a git repo). + + Example: 0.0.0-gitbf01c19 + + :return: version string + """ + + version = get_package_version() + commit = get_git_commit() + if commit: + version = version + "-git" + commit[:7] + + return version + + +__all__ = ["get_version"] diff --git a/core/db/__init__.py b/core/db/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/core/db/alembic.ini b/core/db/alembic.ini new file mode 100644 index 00000000..667ce857 --- /dev/null +++ b/core/db/alembic.ini @@ -0,0 +1,116 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = core/db/migrations + +# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# Uncomment the line below if you want the files to be prepended with date and time +# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file +# for all available tokens +# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the python>=3.9 or backports.zoneinfo library. +# Any required deps can installed by adding `alembic[tz]` to the pip requirements +# string value is passed to ZoneInfo() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to migrations/versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "version_path_separator" below. +version_locations = core/db/migrations/versions + +# version path separator; As mentioned above, this is the character used to split +# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. +# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. +# Valid values for version_path_separator are: +# +# version_path_separator = : +# version_path_separator = ; +# version_path_separator = space +# Use os.pathsep. Default configuration used for new projects. +version_path_separator = os + +# set to 'true' to search source files recursively +# in each "version_locations" directory +# new in Alembic version 1.10 +# recursive_version_locations = false + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = sqlite:///data/database/pythagora.db + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# lint with attempts to fix using "ruff" - use the exec runner, execute a binary +hooks = ruff +ruff.type = exec +ruff.executable = ruff +ruff.options = check --fix REVISION_SCRIPT_FILENAME + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/core/db/migrations/README b/core/db/migrations/README new file mode 100644 index 00000000..5cdc7408 --- /dev/null +++ b/core/db/migrations/README @@ -0,0 +1,10 @@ +Pythagora uses Alembic for database migrations. + +After changing any of the database models, create a new migration: + + alembic -c core/db/alembic.ini revision --autogenerate -m "description" + +Migrations are applied automatically when the application starts, but can also be +run manually with: + + alembic -c core/db/alembic.ini upgrade head diff --git a/core/db/migrations/env.py b/core/db/migrations/env.py new file mode 100644 index 00000000..aea84ff5 --- /dev/null +++ b/core/db/migrations/env.py @@ -0,0 +1,83 @@ +from logging.config import fileConfig + +from alembic import context +from sqlalchemy import engine_from_config, pool + +from core.db.models import Base + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +if config.config_file_name is not None and not config.get_main_option("pythagora_runtime"): + fileConfig(config.config_file_name) + +# Set database URL from environment +# config.set_main_option("sqlalchemy.url", getenv("DATABASE_URL")) + +# add your model's MetaData object here +# for 'autogenerate' support +target_metadata = Base.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + render_as_batch="sqlite://" in url, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + url = config.get_main_option("sqlalchemy.url") + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata, + render_as_batch="sqlite://" in url, + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/core/db/migrations/script.py.mako b/core/db/migrations/script.py.mako new file mode 100644 index 00000000..fbc4b07d --- /dev/null +++ b/core/db/migrations/script.py.mako @@ -0,0 +1,26 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/core/db/migrations/versions/08d71952ec2f_refactor_specification_template_to_.py b/core/db/migrations/versions/08d71952ec2f_refactor_specification_template_to_.py new file mode 100644 index 00000000..655466bd --- /dev/null +++ b/core/db/migrations/versions/08d71952ec2f_refactor_specification_template_to_.py @@ -0,0 +1,36 @@ +"""refactor specification.template to specification.templates + +Revision ID: 08d71952ec2f +Revises: ff891d366761 +Create Date: 2024-06-14 18:23:09.070736 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "08d71952ec2f" +down_revision: Union[str, None] = "ff891d366761" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("specifications", schema=None) as batch_op: + batch_op.add_column(sa.Column("templates", sa.JSON(), nullable=True)) + batch_op.drop_column("template") + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("specifications", schema=None) as batch_op: + batch_op.add_column(sa.Column("template", sa.VARCHAR(), nullable=True)) + batch_op.drop_column("templates") + + # ### end Alembic commands ### diff --git a/core/db/migrations/versions/0a1bb637fa26_initial.py b/core/db/migrations/versions/0a1bb637fa26_initial.py new file mode 100644 index 00000000..e7172daf --- /dev/null +++ b/core/db/migrations/versions/0a1bb637fa26_initial.py @@ -0,0 +1,202 @@ +"""initial + +Revision ID: 0a1bb637fa26 +Revises: +Create Date: 2024-05-28 09:49:51.582784 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "0a1bb637fa26" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "file_contents", + sa.Column("id", sa.String(), nullable=False), + sa.Column("content", sa.String(), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_file_contents")), + ) + op.create_table( + "projects", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.Column("created_at", sa.DateTime(), server_default=sa.text("(CURRENT_TIMESTAMP)"), nullable=False), + sa.Column("folder_name", sa.String(), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_projects")), + ) + op.create_table( + "specifications", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("description", sa.String(), nullable=False), + sa.Column("architecture", sa.String(), nullable=False), + sa.Column("system_dependencies", sa.JSON(), nullable=False), + sa.Column("package_dependencies", sa.JSON(), nullable=False), + sa.Column("template", sa.String(), nullable=True), + sa.Column("complexity", sa.String(), server_default="hard", nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_specifications")), + ) + op.create_table( + "branches", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("project_id", sa.Uuid(), nullable=False), + sa.Column("created_at", sa.DateTime(), server_default=sa.text("(CURRENT_TIMESTAMP)"), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.ForeignKeyConstraint( + ["project_id"], ["projects.id"], name=op.f("fk_branches_project_id_projects"), ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_branches")), + ) + op.create_table( + "project_states", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("branch_id", sa.Uuid(), nullable=False), + sa.Column("prev_state_id", sa.Uuid(), nullable=True), + sa.Column("specification_id", sa.Integer(), nullable=False), + sa.Column("created_at", sa.DateTime(), server_default=sa.text("(CURRENT_TIMESTAMP)"), nullable=False), + sa.Column("step_index", sa.Integer(), server_default="1", nullable=False), + sa.Column("epics", sa.JSON(), nullable=False), + sa.Column("tasks", sa.JSON(), nullable=False), + sa.Column("steps", sa.JSON(), nullable=False), + sa.Column("iterations", sa.JSON(), nullable=False), + sa.Column("relevant_files", sa.JSON(), nullable=False), + sa.Column("modified_files", sa.JSON(), nullable=False), + sa.Column("run_command", sa.String(), nullable=True), + sa.Column("action", sa.String(), nullable=True), + sa.ForeignKeyConstraint( + ["branch_id"], ["branches.id"], name=op.f("fk_project_states_branch_id_branches"), ondelete="CASCADE" + ), + sa.ForeignKeyConstraint( + ["prev_state_id"], + ["project_states.id"], + name=op.f("fk_project_states_prev_state_id_project_states"), + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["specification_id"], ["specifications.id"], name=op.f("fk_project_states_specification_id_specifications") + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_project_states")), + sa.UniqueConstraint("branch_id", "step_index", name=op.f("uq_project_states_branch_id")), + sa.UniqueConstraint("prev_state_id", name=op.f("uq_project_states_prev_state_id")), + sqlite_autoincrement=True, + ) + op.create_table( + "exec_logs", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("branch_id", sa.Uuid(), nullable=False), + sa.Column("project_state_id", sa.Uuid(), nullable=True), + sa.Column("started_at", sa.DateTime(), nullable=False), + sa.Column("duration", sa.Float(), nullable=False), + sa.Column("cmd", sa.String(), nullable=False), + sa.Column("cwd", sa.String(), nullable=False), + sa.Column("env", sa.JSON(), nullable=False), + sa.Column("timeout", sa.Float(), nullable=True), + sa.Column("status_code", sa.Integer(), nullable=True), + sa.Column("stdout", sa.String(), nullable=False), + sa.Column("stderr", sa.String(), nullable=False), + sa.Column("analysis", sa.String(), nullable=False), + sa.Column("success", sa.Boolean(), nullable=False), + sa.ForeignKeyConstraint( + ["branch_id"], ["branches.id"], name=op.f("fk_exec_logs_branch_id_branches"), ondelete="CASCADE" + ), + sa.ForeignKeyConstraint( + ["project_state_id"], + ["project_states.id"], + name=op.f("fk_exec_logs_project_state_id_project_states"), + ondelete="SET NULL", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_exec_logs")), + ) + op.create_table( + "files", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("project_state_id", sa.Uuid(), nullable=False), + sa.Column("content_id", sa.String(), nullable=False), + sa.Column("path", sa.String(), nullable=False), + sa.Column("meta", sa.JSON(), server_default="{}", nullable=False), + sa.ForeignKeyConstraint( + ["content_id"], ["file_contents.id"], name=op.f("fk_files_content_id_file_contents"), ondelete="RESTRICT" + ), + sa.ForeignKeyConstraint( + ["project_state_id"], + ["project_states.id"], + name=op.f("fk_files_project_state_id_project_states"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_files")), + sa.UniqueConstraint("project_state_id", "path", name=op.f("uq_files_project_state_id")), + ) + op.create_table( + "llm_requests", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("branch_id", sa.Uuid(), nullable=False), + sa.Column("project_state_id", sa.Uuid(), nullable=True), + sa.Column("started_at", sa.DateTime(), server_default=sa.text("(CURRENT_TIMESTAMP)"), nullable=False), + sa.Column("agent", sa.String(), nullable=True), + sa.Column("provider", sa.String(), nullable=False), + sa.Column("model", sa.String(), nullable=False), + sa.Column("temperature", sa.Float(), nullable=False), + sa.Column("messages", sa.JSON(), nullable=False), + sa.Column("prompts", sa.JSON(), server_default="[]", nullable=False), + sa.Column("response", sa.String(), nullable=True), + sa.Column("prompt_tokens", sa.Integer(), nullable=False), + sa.Column("completion_tokens", sa.Integer(), nullable=False), + sa.Column("duration", sa.Float(), nullable=False), + sa.Column("status", sa.String(), nullable=False), + sa.Column("error", sa.String(), nullable=True), + sa.ForeignKeyConstraint( + ["branch_id"], ["branches.id"], name=op.f("fk_llm_requests_branch_id_branches"), ondelete="CASCADE" + ), + sa.ForeignKeyConstraint( + ["project_state_id"], + ["project_states.id"], + name=op.f("fk_llm_requests_project_state_id_project_states"), + ondelete="SET NULL", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_llm_requests")), + ) + op.create_table( + "user_inputs", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("branch_id", sa.Uuid(), nullable=False), + sa.Column("project_state_id", sa.Uuid(), nullable=True), + sa.Column("created_at", sa.DateTime(), server_default=sa.text("(CURRENT_TIMESTAMP)"), nullable=False), + sa.Column("question", sa.String(), nullable=False), + sa.Column("answer_text", sa.String(), nullable=True), + sa.Column("answer_button", sa.String(), nullable=True), + sa.Column("cancelled", sa.Boolean(), nullable=False), + sa.ForeignKeyConstraint( + ["branch_id"], ["branches.id"], name=op.f("fk_user_inputs_branch_id_branches"), ondelete="CASCADE" + ), + sa.ForeignKeyConstraint( + ["project_state_id"], + ["project_states.id"], + name=op.f("fk_user_inputs_project_state_id_project_states"), + ondelete="SET NULL", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_user_inputs")), + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("user_inputs") + op.drop_table("llm_requests") + op.drop_table("files") + op.drop_table("exec_logs") + op.drop_table("project_states") + op.drop_table("branches") + op.drop_table("specifications") + op.drop_table("projects") + op.drop_table("file_contents") + # ### end Alembic commands ### diff --git a/core/db/migrations/versions/b760f66138c0_add_docs_column_to_project_states.py b/core/db/migrations/versions/b760f66138c0_add_docs_column_to_project_states.py new file mode 100644 index 00000000..5ab1ee59 --- /dev/null +++ b/core/db/migrations/versions/b760f66138c0_add_docs_column_to_project_states.py @@ -0,0 +1,34 @@ +"""Add docs column to project_states + +Revision ID: b760f66138c0 +Revises: f352dbe45751 +Create Date: 2024-06-08 10:00:44.222099 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "b760f66138c0" +down_revision: Union[str, None] = "f352dbe45751" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("project_states", schema=None) as batch_op: + batch_op.add_column(sa.Column("docs", sa.JSON(), nullable=True)) + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("project_states", schema=None) as batch_op: + batch_op.drop_column("docs") + + # ### end Alembic commands ### diff --git a/core/db/migrations/versions/c8905d4ce784_add_original_description_and_template_.py b/core/db/migrations/versions/c8905d4ce784_add_original_description_and_template_.py new file mode 100644 index 00000000..5d76dba8 --- /dev/null +++ b/core/db/migrations/versions/c8905d4ce784_add_original_description_and_template_.py @@ -0,0 +1,36 @@ +"""Add original description and template summary fields to specifications + +Revision ID: c8905d4ce784 +Revises: 08d71952ec2f +Create Date: 2024-07-25 19:24:23.808237 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "c8905d4ce784" +down_revision: Union[str, None] = "08d71952ec2f" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("specifications", schema=None) as batch_op: + batch_op.add_column(sa.Column("original_description", sa.String(), nullable=True)) + batch_op.add_column(sa.Column("template_summary", sa.String(), nullable=True)) + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("specifications", schema=None) as batch_op: + batch_op.drop_column("template_summary") + batch_op.drop_column("original_description") + + # ### end Alembic commands ### diff --git a/core/db/migrations/versions/f352dbe45751_make_relevant_files_nullable.py b/core/db/migrations/versions/f352dbe45751_make_relevant_files_nullable.py new file mode 100644 index 00000000..b42f94a5 --- /dev/null +++ b/core/db/migrations/versions/f352dbe45751_make_relevant_files_nullable.py @@ -0,0 +1,34 @@ +"""Make relevant_files nullable + +Revision ID: f352dbe45751 +Revises: 0a1bb637fa26 +Create Date: 2024-06-04 15:07:40.175466 + +""" + +from typing import Sequence, Union + +from alembic import op +from sqlalchemy.dialects import sqlite + +# revision identifiers, used by Alembic. +revision: str = "f352dbe45751" +down_revision: Union[str, None] = "0a1bb637fa26" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("project_states", schema=None) as batch_op: + batch_op.alter_column("relevant_files", existing_type=sqlite.JSON(), nullable=True) + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("project_states", schema=None) as batch_op: + batch_op.alter_column("relevant_files", existing_type=sqlite.JSON(), nullable=False) + + # ### end Alembic commands ### diff --git a/core/db/migrations/versions/f708791b9270_adding_knowledge_base_field_to_.py b/core/db/migrations/versions/f708791b9270_adding_knowledge_base_field_to_.py new file mode 100644 index 00000000..f19ee00e --- /dev/null +++ b/core/db/migrations/versions/f708791b9270_adding_knowledge_base_field_to_.py @@ -0,0 +1,34 @@ +"""Adding knowledge_base field to ProjectState + +Revision ID: f708791b9270 +Revises: c8905d4ce784 +Create Date: 2024-12-22 12:13:14.979169 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "f708791b9270" +down_revision: Union[str, None] = "c8905d4ce784" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("project_states", schema=None) as batch_op: + batch_op.add_column(sa.Column("knowledge_base", sa.JSON(), server_default="{}", nullable=False)) + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("project_states", schema=None) as batch_op: + batch_op.drop_column("knowledge_base") + + # ### end Alembic commands ### diff --git a/core/db/migrations/versions/ff891d366761_add_example_project_to_spec.py b/core/db/migrations/versions/ff891d366761_add_example_project_to_spec.py new file mode 100644 index 00000000..f788761e --- /dev/null +++ b/core/db/migrations/versions/ff891d366761_add_example_project_to_spec.py @@ -0,0 +1,34 @@ +"""add example project to spec + +Revision ID: ff891d366761 +Revises: b760f66138c0 +Create Date: 2024-06-13 09:38:33.329161 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "ff891d366761" +down_revision: Union[str, None] = "b760f66138c0" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("specifications", schema=None) as batch_op: + batch_op.add_column(sa.Column("example_project", sa.String(), nullable=True)) + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("specifications", schema=None) as batch_op: + batch_op.drop_column("example_project") + + # ### end Alembic commands ### diff --git a/core/db/models/__init__.py b/core/db/models/__init__.py new file mode 100644 index 00000000..1b81fb88 --- /dev/null +++ b/core/db/models/__init__.py @@ -0,0 +1,29 @@ +# Pythagora database models +# +# Always import models from this module to ensure the SQLAlchemy registry +# is correctly populated. + +from .base import Base +from .branch import Branch +from .exec_log import ExecLog +from .file import File +from .file_content import FileContent +from .llm_request import LLMRequest +from .project import Project +from .project_state import ProjectState +from .specification import Complexity, Specification +from .user_input import UserInput + +__all__ = [ + "Base", + "Branch", + "Complexity", + "ExecLog", + "File", + "FileContent", + "LLMRequest", + "Project", + "ProjectState", + "Specification", + "UserInput", +] diff --git a/core/db/models/base.py b/core/db/models/base.py new file mode 100644 index 00000000..5ef25859 --- /dev/null +++ b/core/db/models/base.py @@ -0,0 +1,36 @@ +# DeclarativeBase enables declarative configuration of +# database models within SQLAlchemy. +# +# It also sets up a registry for the classes that inherit from it, +# so that SQLAlechemy understands how they map to database tables. + +from sqlalchemy import MetaData +from sqlalchemy.ext.asyncio import AsyncAttrs +from sqlalchemy.orm import DeclarativeBase +from sqlalchemy.types import JSON + + +class Base(AsyncAttrs, DeclarativeBase): + """Base class for all SQL database models.""" + + # Mapping of Python types to SQLAlchemy types. + type_annotation_map = { + list[dict]: JSON, + list[str]: JSON, + dict: JSON, + } + + metadata = MetaData( + # Naming conventions for constraints, foreign keys, etc. + naming_convention={ + "ix": "ix_%(column_0_label)s", + "uq": "uq_%(table_name)s_%(column_0_name)s", + "ck": "ck_%(table_name)s_`%(constraint_name)s`", + "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s", + "pk": "pk_%(table_name)s", + } + ) + + def __repr__(self) -> str: + """Return a string representation of the model.""" + return f"<{self.__class__.__name__}(id={self.id})>" diff --git a/core/db/models/branch.py b/core/db/models/branch.py new file mode 100644 index 00000000..925da033 --- /dev/null +++ b/core/db/models/branch.py @@ -0,0 +1,89 @@ +from datetime import datetime +from typing import TYPE_CHECKING, Optional, Union +from uuid import UUID, uuid4 + +from sqlalchemy import ForeignKey, inspect, select +from sqlalchemy.orm import Mapped, mapped_column, relationship +from sqlalchemy.sql import func + +from core.db.models import Base + +if TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession + + from core.db.models import ExecLog, LLMRequest, Project, ProjectState, UserInput + + +class Branch(Base): + __tablename__ = "branches" + + DEFAULT = "main" + + # ID and parent FKs + id: Mapped[UUID] = mapped_column(primary_key=True, default=uuid4) + project_id: Mapped[UUID] = mapped_column(ForeignKey("projects.id", ondelete="CASCADE")) + + # Attributes + created_at: Mapped[datetime] = mapped_column(server_default=func.now()) + name: Mapped[str] = mapped_column(default=DEFAULT) + + # Relationships + project: Mapped["Project"] = relationship(back_populates="branches", lazy="selectin") + states: Mapped[list["ProjectState"]] = relationship(back_populates="branch", cascade="all", lazy="raise") + llm_requests: Mapped[list["LLMRequest"]] = relationship(back_populates="branch", cascade="all", lazy="raise") + user_inputs: Mapped[list["UserInput"]] = relationship(back_populates="branch", cascade="all", lazy="raise") + exec_logs: Mapped[list["ExecLog"]] = relationship(back_populates="branch", cascade="all", lazy="raise") + + @staticmethod + async def get_by_id(session: "AsyncSession", branch_id: Union[str, UUID]) -> Optional["Branch"]: + """ + Get a project by ID. + + :param session: The SQLAlchemy session. + :param project_id: The branch ID (as str or UUID value). + :return: The Branch object if found, None otherwise. + """ + if not isinstance(branch_id, UUID): + branch_id = UUID(branch_id) + + result = await session.execute(select(Branch).where(Branch.id == branch_id)) + return result.scalar_one_or_none() + + async def get_last_state(self) -> Optional["ProjectState"]: + """ + Get the last project state of the branch. + + :return: The last step of the branch, or None if there are no steps. + """ + + from core.db.models import ProjectState + + session = inspect(self).async_session + if session is None: + raise ValueError("Branch instance not associated with a DB session.") + + result = await session.execute( + select(ProjectState) + .where(ProjectState.branch_id == self.id) + .order_by(ProjectState.step_index.desc()) + .limit(1) + ) + return result.scalar_one_or_none() + + async def get_state_at_step(self, step_index: int) -> Optional["ProjectState"]: + """ + Get the project state at the given step index for the branch. + + :return: The indicated step within the branch, or None if there's no such step. + """ + + from core.db.models import ProjectState + + session = inspect(self).async_session + if session is None: + raise ValueError("Branch instance not associated with a DB session.") + + result = await session.execute( + select(ProjectState).where((ProjectState.branch_id == self.id) & (ProjectState.step_index == step_index)) + ) + return result.scalar_one_or_none() diff --git a/core/db/models/exec_log.py b/core/db/models/exec_log.py new file mode 100644 index 00000000..4c543e6f --- /dev/null +++ b/core/db/models/exec_log.py @@ -0,0 +1,72 @@ +from datetime import datetime +from typing import TYPE_CHECKING, Optional +from uuid import UUID + +from sqlalchemy import ForeignKey, inspect +from sqlalchemy.orm import Mapped, mapped_column, relationship +from sqlalchemy.sql import func + +from core.db.models import Base +from core.proc.exec_log import ExecLog as ExecLogData + +if TYPE_CHECKING: + from core.db.models import Branch, ProjectState + + +class ExecLog(Base): + __tablename__ = "exec_logs" + + # ID and parent FKs + id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True) + branch_id: Mapped[UUID] = mapped_column(ForeignKey("branches.id", ondelete="CASCADE")) + project_state_id: Mapped[Optional[UUID]] = mapped_column(ForeignKey("project_states.id", ondelete="SET NULL")) + + # Attributes + started_at: Mapped[datetime] = mapped_column(server_default=func.now()) + duration: Mapped[float] = mapped_column() + cmd: Mapped[str] = mapped_column() + cwd: Mapped[str] = mapped_column() + env: Mapped[dict] = mapped_column() + timeout: Mapped[Optional[float]] = mapped_column() + status_code: Mapped[Optional[int]] = mapped_column() + stdout: Mapped[str] = mapped_column() + stderr: Mapped[str] = mapped_column() + analysis: Mapped[str] = mapped_column() + success: Mapped[bool] = mapped_column() + + # Relationships + branch: Mapped["Branch"] = relationship(back_populates="exec_logs", lazy="raise") + project_state: Mapped["ProjectState"] = relationship(back_populates="exec_logs", lazy="raise") + + @classmethod + def from_exec_log(cls, project_state: "ProjectState", exec_log: ExecLogData) -> "ExecLog": + """ + Store the user input in the database. + + Note this just creates the UserInput object. It is committed to the + database only when the DB session itself is comitted. + + :param project_state: Project state to associate the request log with. + :param question: Question the user was asked. + :param user_input: User input. + :return: Newly created User input in the database. + """ + session = inspect(project_state).async_session + + obj = cls( + project_state=project_state, + branch=project_state.branch, + started_at=exec_log.started_at, + duration=exec_log.duration, + cmd=exec_log.cmd, + cwd=exec_log.cwd, + env=exec_log.env, + timeout=exec_log.timeout, + status_code=exec_log.status_code, + stdout=exec_log.stdout, + stderr=exec_log.stderr, + analysis=exec_log.analysis, + success=exec_log.success, + ) + session.add(obj) + return obj diff --git a/core/db/models/file.py b/core/db/models/file.py new file mode 100644 index 00000000..d75ee68f --- /dev/null +++ b/core/db/models/file.py @@ -0,0 +1,43 @@ +from typing import TYPE_CHECKING, Optional +from uuid import UUID + +from sqlalchemy import ForeignKey, UniqueConstraint +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from core.db.models import Base + +if TYPE_CHECKING: + from core.db.models import FileContent, ProjectState + + +class File(Base): + __tablename__ = "files" + __table_args__ = (UniqueConstraint("project_state_id", "path"),) + + # ID and parent FKs + id: Mapped[int] = mapped_column(primary_key=True) + project_state_id: Mapped[UUID] = mapped_column(ForeignKey("project_states.id", ondelete="CASCADE")) + content_id: Mapped[str] = mapped_column(ForeignKey("file_contents.id", ondelete="RESTRICT")) + + # Attributes + path: Mapped[str] = mapped_column() + meta: Mapped[dict] = mapped_column(default=dict, server_default="{}") + + # Relationships + project_state: Mapped[Optional["ProjectState"]] = relationship(back_populates="files", lazy="raise") + content: Mapped["FileContent"] = relationship(back_populates="files", lazy="selectin") + + def clone(self) -> "File": + """ + Clone the file object, to be used in a new project state. + + The clone references the same file content object as the original. + + :return: The cloned file object. + """ + return File( + project_state=None, + content_id=self.content_id, + path=self.path, + meta=self.meta, + ) diff --git a/core/db/models/file_content.py b/core/db/models/file_content.py new file mode 100644 index 00000000..57cbe8c3 --- /dev/null +++ b/core/db/models/file_content.py @@ -0,0 +1,58 @@ +from typing import TYPE_CHECKING + +from sqlalchemy import delete, distinct, select +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from core.db.models import Base + +if TYPE_CHECKING: + from core.db.models import File + + +class FileContent(Base): + __tablename__ = "file_contents" + + # ID and parent FKs + id: Mapped[str] = mapped_column(primary_key=True) + + # Attributes + content: Mapped[str] = mapped_column() + + # Relationships + files: Mapped[list["File"]] = relationship(back_populates="content", lazy="raise") + + @classmethod + async def store(cls, session: AsyncSession, hash: str, content: str) -> "FileContent": + """ + Store the file content in the database. + + If the content is already stored, returns the reference to the existing + content object. Otherwise stores it to the database and returns the newly + created content object. + + :param session: The database session. + :param hash: The hash of the file content, used as an unique ID. + :param content: The file content as unicode string. + :return: The file content object. + """ + result = await session.execute(select(FileContent).where(FileContent.id == hash)) + fc = result.scalar_one_or_none() + if fc is not None: + return fc + + fc = cls(id=hash, content=content) + session.add(fc) + + return fc + + @classmethod + async def delete_orphans(cls, session: AsyncSession): + """ + Delete FileContent objects that are not referenced by any File object. + + :param session: The database session. + """ + from core.db.models import File + + await session.execute(delete(FileContent).where(~FileContent.id.in_(select(distinct(File.content_id))))) diff --git a/core/db/models/llm_request.py b/core/db/models/llm_request.py new file mode 100644 index 00000000..9738a176 --- /dev/null +++ b/core/db/models/llm_request.py @@ -0,0 +1,82 @@ +from datetime import datetime +from typing import TYPE_CHECKING, Optional +from uuid import UUID + +from sqlalchemy import ForeignKey, inspect +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import Mapped, mapped_column, relationship +from sqlalchemy.sql import func + +from core.db.models import Base +from core.llm.request_log import LLMRequestLog + +if TYPE_CHECKING: + from core.agents.base import BaseAgent + from core.db.models import Branch, ProjectState + + +class LLMRequest(Base): + __tablename__ = "llm_requests" + + # ID and parent FKs + id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True) + branch_id: Mapped[UUID] = mapped_column(ForeignKey("branches.id", ondelete="CASCADE")) + project_state_id: Mapped[Optional[UUID]] = mapped_column(ForeignKey("project_states.id", ondelete="SET NULL")) + + # Attributes + started_at: Mapped[datetime] = mapped_column(server_default=func.now()) + agent: Mapped[Optional[str]] = mapped_column() + provider: Mapped[str] = mapped_column() + model: Mapped[str] = mapped_column() + temperature: Mapped[float] = mapped_column() + messages: Mapped[list[dict]] = mapped_column() + prompts: Mapped[list[str]] = mapped_column(server_default="[]") + response: Mapped[Optional[str]] = mapped_column() + prompt_tokens: Mapped[int] = mapped_column() + completion_tokens: Mapped[int] = mapped_column() + duration: Mapped[float] = mapped_column() + status: Mapped[str] = mapped_column() + error: Mapped[Optional[str]] = mapped_column() + + # Relationships + branch: Mapped["Branch"] = relationship(back_populates="llm_requests", lazy="raise") + project_state: Mapped["ProjectState"] = relationship(back_populates="llm_requests", lazy="raise") + + @classmethod + def from_request_log( + cls, + project_state: "ProjectState", + agent: Optional["BaseAgent"], + request_log: LLMRequestLog, + ) -> "LLMRequest": + """ + Store the request log in the database. + + Note this just creates the request log object. It is committed to the + database only when the DB session itself is comitted. + + :param project_state: Project state to associate the request log with. + :param agent: Agent that made the request (if the caller was an agent). + :param request_log: Request log. + :return: Newly created LLM request log in the database. + """ + session: AsyncSession = inspect(project_state).async_session + + obj = cls( + project_state=project_state, + branch=project_state.branch, + agent=agent.agent_type, + provider=request_log.provider, + model=request_log.model, + temperature=request_log.temperature, + messages=request_log.messages, + prompts=request_log.prompts, + response=request_log.response, + prompt_tokens=request_log.prompt_tokens, + completion_tokens=request_log.completion_tokens, + duration=request_log.duration, + status=request_log.status, + error=request_log.error, + ) + session.add(obj) + return obj diff --git a/core/db/models/project.py b/core/db/models/project.py new file mode 100644 index 00000000..4f1a716a --- /dev/null +++ b/core/db/models/project.py @@ -0,0 +1,130 @@ +import re +from datetime import datetime +from typing import TYPE_CHECKING, Optional, Union +from unicodedata import normalize +from uuid import UUID, uuid4 + +from sqlalchemy import and_, delete, inspect, select +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import Mapped, mapped_column, relationship, selectinload +from sqlalchemy.sql import func + +from core.db.models import Base + +if TYPE_CHECKING: + from core.db.models import Branch + + +class Project(Base): + __tablename__ = "projects" + + # ID and parent FKs + id: Mapped[UUID] = mapped_column(primary_key=True, default=uuid4) + + # Attributes + name: Mapped[str] = mapped_column() + created_at: Mapped[datetime] = mapped_column(server_default=func.now()) + folder_name: Mapped[str] = mapped_column( + default=lambda context: Project.get_folder_from_project_name(context.get_current_parameters()["name"]) + ) + + # Relationships + branches: Mapped[list["Branch"]] = relationship(back_populates="project", cascade="all", lazy="raise") + + @staticmethod + async def get_by_id(session: "AsyncSession", project_id: Union[str, UUID]) -> Optional["Project"]: + """ + Get a project by ID. + + :param session: The SQLAlchemy session. + :param project_id: The project ID (as str or UUID value). + :return: The Project object if found, None otherwise. + """ + if not isinstance(project_id, UUID): + project_id = UUID(project_id) + + result = await session.execute(select(Project).where(Project.id == project_id)) + return result.scalar_one_or_none() + + async def get_branch(self, name: Optional[str] = None) -> Optional["Branch"]: + """ + Get a project branch by name. + + :param session: The SQLAlchemy session. + :param branch_name: The name of the branch (default "main"). + :return: The Branch object if found, None otherwise. + """ + from core.db.models import Branch + + session = inspect(self).async_session + if session is None: + raise ValueError("Project instance not associated with a DB session.") + + if name is None: + name = Branch.DEFAULT + + result = await session.execute(select(Branch).where(Branch.project_id == self.id, Branch.name == name)) + return result.scalar_one_or_none() + + @staticmethod + async def get_all_projects(session: "AsyncSession") -> list["Project"]: + """ + Get all projects. + + This assumes the projects have at least one branch and one state. + + :param session: The SQLAlchemy session. + :return: List of Project objects. + """ + from core.db.models import Branch, ProjectState + + latest_state_query = ( + select(ProjectState.branch_id, func.max(ProjectState.step_index).label("max_index")) + .group_by(ProjectState.branch_id) + .subquery() + ) + + query = ( + select(Project, Branch, ProjectState) + .join(Branch, Project.branches) + .join(ProjectState, Branch.states) + .join( + latest_state_query, + and_( + ProjectState.branch_id == latest_state_query.columns.branch_id, + ProjectState.step_index == latest_state_query.columns.max_index, + ), + ) + .options(selectinload(Project.branches), selectinload(Branch.states)) + .order_by(Project.name, Branch.name) + ) + + results = await session.execute(query) + return results.scalars().all() + + @staticmethod + def get_folder_from_project_name(name: str): + """ + Get the folder name from the project name. + + :param name: Project name. + :return: Folder name. + """ + # replace unicode with accents with base characters (eg "Ε‘aΕ‘avi" β†’ "sasavi") + name = normalize("NFKD", name).encode("ascii", "ignore").decode("utf-8") + + # replace spaces/interpunction with a single dash + return re.sub(r"[^a-zA-Z0-9]+", "-", name).lower().strip("-") + + @staticmethod + async def delete_by_id(session: "AsyncSession", project_id: UUID) -> int: + """ + Delete a project by ID. + + :param session: The SQLAlchemy session. + :param project_id: The project ID + :return: Number of rows deleted. + """ + + result = await session.execute(delete(Project).where(Project.id == project_id)) + return result.rowcount diff --git a/core/db/models/project_state.py b/core/db/models/project_state.py new file mode 100644 index 00000000..a68a4aa8 --- /dev/null +++ b/core/db/models/project_state.py @@ -0,0 +1,495 @@ +from copy import deepcopy +from datetime import datetime +from typing import TYPE_CHECKING, Optional, Union +from uuid import UUID, uuid4 + +from sqlalchemy import ForeignKey, UniqueConstraint, delete, inspect +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import Mapped, mapped_column, relationship +from sqlalchemy.orm.attributes import flag_modified +from sqlalchemy.sql import func + +from core.db.models import Base, FileContent +from core.log import get_logger + +if TYPE_CHECKING: + from core.db.models import Branch, ExecLog, File, FileContent, LLMRequest, Specification, UserInput + +log = get_logger(__name__) + + +class TaskStatus: + """Status of a task.""" + + TODO = "todo" + IN_PROGRESS = "in_progress" + REVIEWED = "reviewed" + DOCUMENTED = "documented" + EPIC_UPDATED = "epic_updated" + DONE = "done" + SKIPPED = "skipped" + + +class IterationStatus: + """Status of an iteration.""" + + HUNTING_FOR_BUG = "check_logs" + AWAITING_LOGGING = "awaiting_logging" + AWAITING_USER_TEST = "awaiting_user_test" + AWAITING_BUG_FIX = "awaiting_bug_fix" + AWAITING_BUG_REPRODUCTION = "awaiting_bug_reproduction" + IMPLEMENT_SOLUTION = "implement_solution" + FIND_SOLUTION = "find_solution" + PROBLEM_SOLVER = "problem_solver" + NEW_FEATURE_REQUESTED = "new_feature_requested" + START_PAIR_PROGRAMMING = "start_pair_programming" + DONE = "done" + + +class ProjectState(Base): + __tablename__ = "project_states" + __table_args__ = ( + UniqueConstraint("prev_state_id"), + UniqueConstraint("branch_id", "step_index"), + {"sqlite_autoincrement": True}, + ) + + # ID and parent FKs + id: Mapped[UUID] = mapped_column(primary_key=True, default=uuid4) + branch_id: Mapped[UUID] = mapped_column(ForeignKey("branches.id", ondelete="CASCADE")) + prev_state_id: Mapped[Optional[UUID]] = mapped_column(ForeignKey("project_states.id", ondelete="CASCADE")) + specification_id: Mapped[int] = mapped_column(ForeignKey("specifications.id")) + + # Attributes + created_at: Mapped[datetime] = mapped_column(server_default=func.now()) + step_index: Mapped[int] = mapped_column(default=1, server_default="1") + epics: Mapped[list[dict]] = mapped_column(default=list) + tasks: Mapped[list[dict]] = mapped_column(default=list) + steps: Mapped[list[dict]] = mapped_column(default=list) + iterations: Mapped[list[dict]] = mapped_column(default=list) + knowledge_base: Mapped[dict] = mapped_column(default=dict, server_default="{}") + relevant_files: Mapped[Optional[list[str]]] = mapped_column(default=None) + modified_files: Mapped[dict] = mapped_column(default=dict) + docs: Mapped[Optional[list[dict]]] = mapped_column(default=None) + run_command: Mapped[Optional[str]] = mapped_column() + action: Mapped[Optional[str]] = mapped_column() + + # Relationships + branch: Mapped["Branch"] = relationship(back_populates="states", lazy="selectin") + prev_state: Mapped[Optional["ProjectState"]] = relationship( + back_populates="next_state", + remote_side=[id], + single_parent=True, + lazy="raise", + cascade="delete", + ) + next_state: Mapped[Optional["ProjectState"]] = relationship(back_populates="prev_state", lazy="raise") + files: Mapped[list["File"]] = relationship( + back_populates="project_state", + lazy="selectin", + cascade="all,delete-orphan", + ) + specification: Mapped["Specification"] = relationship(back_populates="project_states", lazy="selectin") + llm_requests: Mapped[list["LLMRequest"]] = relationship(back_populates="project_state", cascade="all", lazy="raise") + user_inputs: Mapped[list["UserInput"]] = relationship(back_populates="project_state", cascade="all", lazy="raise") + exec_logs: Mapped[list["ExecLog"]] = relationship(back_populates="project_state", cascade="all", lazy="raise") + + @property + def unfinished_steps(self) -> list[dict]: + """ + Get the list of unfinished steps. + + :return: List of unfinished steps. + """ + return [step for step in self.steps if not step.get("completed")] + + @property + def current_step(self) -> Optional[dict]: + """ + Get the current step. + + Current step is always the first step that's not finished yet. + + :return: The current step, or None if there are no more unfinished steps. + """ + li = self.unfinished_steps + return li[0] if li else None + + @property + def unfinished_iterations(self) -> list[dict]: + """ + Get the list of unfinished iterations. + + :return: List of unfinished iterations. + """ + return [ + iteration for iteration in self.iterations if iteration.get("status") not in (None, IterationStatus.DONE) + ] + + @property + def current_iteration(self) -> Optional[dict]: + """ + Get the current iteration. + + Current iteration is always the first iteration that's not finished yet. + + :return: The current iteration, or None if there are no unfinished iterations. + """ + li = self.unfinished_iterations + return li[0] if li else None + + @property + def unfinished_tasks(self) -> list[dict]: + """ + Get the list of unfinished tasks. + + :return: List of unfinished tasks. + """ + return [task for task in self.tasks if task.get("status") != TaskStatus.DONE] + + @property + def current_task(self) -> Optional[dict]: + """ + Get the current task. + + Current task is always the first task that's not finished yet. + + :return: The current task, or None if there are no unfinished tasks. + """ + li = self.unfinished_tasks + return li[0] if li else None + + @property + def unfinished_epics(self) -> list[dict]: + """ + Get the list of unfinished epics. + + :return: List of unfinished epics. + """ + return [epic for epic in self.epics if not epic.get("completed")] + + @property + def current_epic(self) -> Optional[dict]: + """ + Get the current epic. + + Current epic is always the first epic that's not finished yet. + + :return: The current epic, or None if there are no unfinished epics. + """ + li = self.unfinished_epics + return li[0] if li else None + + @property + def relevant_file_objects(self): + """ + Get the relevant files with their content. + + :return: List of tuples with file path and content. + """ + relevant_files = self.relevant_files or [] + modified_files = self.modified_files or {} + + all_files = set(relevant_files + list(modified_files.keys())) + return [file for file in self.files if file.path in all_files] + + @staticmethod + def create_initial_state(branch: "Branch") -> "ProjectState": + """ + Create the initial project state for a new branch. + + This does *not* commit the new state to the database. + + No checks are made to ensure that the branch does not + already have a state. + + :param branch: The branch to create the state for. + :return: The new ProjectState object. + """ + from core.db.models import Specification + + return ProjectState( + branch=branch, + specification=Specification(), + step_index=1, + ) + + async def create_next_state(self) -> "ProjectState": + """ + Create the next project state for the branch. + + This does NOT insert the new state and the associated objects (spec, + files, ...) to the database. + + :param session: The SQLAlchemy session. + :return: The new ProjectState object. + """ + if not self.id: + raise ValueError("Cannot create next state for unsaved state.") + + if "next_state" in self.__dict__: + raise ValueError(f"Next state already exists for state with id={self.id}.") + + new_state = ProjectState( + branch=self.branch, + prev_state=self, + step_index=self.step_index + 1, + specification=self.specification, + epics=deepcopy(self.epics), + tasks=deepcopy(self.tasks), + steps=deepcopy(self.steps), + iterations=deepcopy(self.iterations), + knowledge_base=deepcopy(self.knowledge_base), + files=[], + relevant_files=deepcopy(self.relevant_files), + modified_files=deepcopy(self.modified_files), + docs=deepcopy(self.docs), + run_command=self.run_command, + ) + + session: AsyncSession = inspect(self).async_session + session.add(new_state) + + # NOTE: we only need the await here because of the tests, in live, the + # load_project() and commit() methods on StateManager make sure that + # the the files are eagerly loaded. + for file in await self.awaitable_attrs.files: + clone = file.clone() + new_state.files.append(clone) + + return new_state + + def complete_step(self, step_type: str): + if not self.unfinished_steps: + raise ValueError("There are no unfinished steps to complete") + if "next_state" in self.__dict__: + raise ValueError("Current state is read-only (already has a next state).") + + log.debug(f"Completing step {self.unfinished_steps[0]['type']}") + self.get_steps_of_type(step_type)[0]["completed"] = True + flag_modified(self, "steps") + + def complete_task(self): + if not self.unfinished_tasks: + raise ValueError("There are no unfinished tasks to complete") + if "next_state" in self.__dict__: + raise ValueError("Current state is read-only (already has a next state).") + + log.debug(f"Completing task {self.unfinished_tasks[0]['description']}") + self.set_current_task_status(TaskStatus.DONE) + self.steps = [] + self.iterations = [] + self.relevant_files = None + self.modified_files = {} + self.docs = None + flag_modified(self, "tasks") + + if not self.unfinished_tasks and self.unfinished_epics: + self.complete_epic() + + def complete_epic(self): + if not self.unfinished_epics: + raise ValueError("There are no unfinished epics to complete") + if "next_state" in self.__dict__: + raise ValueError("Current state is read-only (already has a next state).") + + log.debug(f"Completing epic {self.unfinished_epics[0]['name']}") + self.unfinished_epics[0]["completed"] = True + self.tasks = [] + flag_modified(self, "epics") + + def complete_iteration(self): + if not self.unfinished_iterations: + raise ValueError("There are no unfinished iterations to complete") + if "next_state" in self.__dict__: + raise ValueError("Current state is read-only (already has a next state).") + + log.debug(f"Completing iteration {self.unfinished_iterations[0]}") + self.unfinished_iterations[0]["status"] = IterationStatus.DONE + self.relevant_files = None + self.modified_files = {} + self.flag_iterations_as_modified() + + def flag_iterations_as_modified(self): + """ + Flag the iterations field as having been modified + + Used by Agents that perform modifications within the mutable iterations field, + to tell the database that it was modified and should get saved (as SQLalchemy + can't detect changes in mutable fields by itself). + """ + flag_modified(self, "iterations") + + def flag_tasks_as_modified(self): + """ + Flag the tasks field as having been modified + + Used by Agents that perform modifications within the mutable tasks field, + to tell the database that it was modified and should get saved (as SQLalchemy + can't detect changes in mutable fields by itself). + """ + flag_modified(self, "tasks") + + def flag_epics_as_modified(self): + """ + Flag the epic field as having been modified + + Used by Agents that perform modifications within the mutable epics field, + to tell the database that it was modified and should get saved (as SQLalchemy + can't detect changes in mutable fields by itself). + """ + flag_modified(self, "epics") + + def flag_knowledge_base_as_modified(self): + """ + Flag the knowledge base field as having been modified + + Used by Agents that perform modifications within the mutable knowledge base field, + to tell the database that it was modified and should get saved (as SQLalchemy + can't detect changes in mutable fields by itself). + """ + flag_modified(self, "knowledge_base") + + def set_current_task_status(self, status: str): + """ + Set the status of the current task. + + :param status: The new status. + """ + if not self.current_task: + raise ValueError("There is no current task to set status for") + if "next_state" in self.__dict__: + raise ValueError("Current state is read-only (already has a next state).") + + self.current_task["status"] = status + self.flag_tasks_as_modified() + + def get_file_by_path(self, path: str) -> Optional["File"]: + """ + Get a file from the current project state, by the file path. + + :param path: The file path. + :return: The file object, or None if not found. + """ + for file in self.files: + if file.path == path: + return file + + return None + + def get_file_content_by_path(self, path: str) -> Union[FileContent, str]: + """ + Get a file from the current project state, by the file path. + + :param path: The file path. + :return: The file object, or None if not found. + """ + file = self.get_file_by_path(path) + + return file.content.content if file else "" + + def save_file(self, path: str, content: "FileContent", external: bool = False) -> "File": + """ + Save a file to the project state. + + This either creates a new file pointing at the given content, + or updates the content of an existing file. This method + doesn't actually commit the file to the database, just attaches + it to the project state. + + If the file was created by Pythagora (not externally by user or template import), + mark it as relevant for the current task. + + :param path: The file path. + :param content: The file content. + :param external: Whether the file was added externally (e.g. by a user). + :return: The (unsaved) file object. + """ + from core.db.models import File + + if "next_state" in self.__dict__: + raise ValueError("Current state is read-only (already has a next state).") + + file = self.get_file_by_path(path) + if file: + original_content = file.content.content + file.content = content + else: + original_content = "" + file = File(path=path, content=content) + self.files.append(file) + + if path not in self.modified_files and not external: + self.modified_files[path] = original_content + + self.relevant_files = self.relevant_files or [] + if path not in self.relevant_files: + self.relevant_files.append(path) + + return file + + async def delete_after(self): + """ + Delete all states in the branch after this one. + """ + + session: AsyncSession = inspect(self).async_session + + log.debug(f"Deleting all project states in branch {self.branch_id} after {self.id}") + await session.execute( + delete(ProjectState).where( + ProjectState.branch_id == self.branch_id, + ProjectState.step_index > self.step_index, + ) + ) + + def get_last_iteration_steps(self) -> list: + """ + Get the steps of the last iteration. + + :return: A list of steps. + """ + return [s for s in self.steps if s.get("iteration_index") == len(self.iterations)] or self.steps + + def get_source_index(self, source: str) -> int: + """ + Get the index of the source which can be one of: 'app', 'feature', 'troubleshooting', 'review'. For example, + for feature return value would be number of current feature. + + :param source: The source to search for. + :return: The index of the source. + """ + if source in ["app", "feature"]: + return len([epic for epic in self.epics if epic.get("source") == source]) + elif source == "troubleshooting": + return len(self.iterations) + elif source == "review": + steps = self.get_last_iteration_steps() + return len([step for step in steps if step.get("type") == "review_task"]) + + return 1 + + def get_steps_of_type(self, step_type: str) -> [dict]: + """ + Get list of unfinished steps with specific type. + + :return: List of steps, or empty list if there are no unfinished steps of that type. + """ + li = self.unfinished_steps + return [step for step in li if step.get("type") == step_type] if li else [] + + def has_frontend(self) -> bool: + """ + Check if there is a frontend epic in the project state. + + :return: True if there is a frontend epic, False otherwise. + """ + return self.epics and any(epic.get("source") == "frontend" for epic in self.epics) + + def is_feature(self) -> bool: + """ + Check if the current epic is a feature. + + :return: True if the current epic is a feature, False otherwise. + """ + return self.epics and self.current_epic and self.current_epic.get("source") == "feature" diff --git a/core/db/models/specification.py b/core/db/models/specification.py new file mode 100644 index 00000000..7631fdeb --- /dev/null +++ b/core/db/models/specification.py @@ -0,0 +1,71 @@ +from copy import deepcopy +from typing import TYPE_CHECKING, Optional + +from sqlalchemy import delete, distinct, select +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from core.db.models import Base + +if TYPE_CHECKING: + from core.db.models import ProjectState + + +class Complexity: + """Estimate of the project or feature complexity.""" + + SIMPLE = "simple" + MODERATE = "moderate" + HARD = "hard" + + +class Specification(Base): + __tablename__ = "specifications" + + # ID and parent FKs + id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True) + + # Attributes + original_description: Mapped[Optional[str]] = mapped_column() + description: Mapped[str] = mapped_column(default="") + template_summary: Mapped[Optional[str]] = mapped_column() + architecture: Mapped[str] = mapped_column(default="") + system_dependencies: Mapped[list[dict]] = mapped_column(default=list) + package_dependencies: Mapped[list[dict]] = mapped_column(default=list) + templates: Mapped[Optional[dict]] = mapped_column() + + complexity: Mapped[str] = mapped_column(server_default=Complexity.HARD) + example_project: Mapped[Optional[str]] = mapped_column() + + # Relationships + project_states: Mapped[list["ProjectState"]] = relationship(back_populates="specification", lazy="raise") + + def clone(self) -> "Specification": + """ + Clone the specification. + """ + clone = Specification( + original_description=self.original_description, + description=self.description, + template_summary=self.template_summary, + architecture=self.architecture, + system_dependencies=self.system_dependencies, + package_dependencies=self.package_dependencies, + templates=deepcopy(self.templates) if self.templates else None, + complexity=self.complexity, + example_project=self.example_project, + ) + return clone + + @classmethod + async def delete_orphans(cls, session: AsyncSession): + """ + Delete Specification objects that are not referenced by any ProjectState object. + + :param session: The database session. + """ + from core.db.models import ProjectState + + await session.execute( + delete(Specification).where(~Specification.id.in_(select(distinct(ProjectState.specification_id)))) + ) diff --git a/core/db/models/user_input.py b/core/db/models/user_input.py new file mode 100644 index 00000000..c068143d --- /dev/null +++ b/core/db/models/user_input.py @@ -0,0 +1,59 @@ +from datetime import datetime +from typing import TYPE_CHECKING, Optional +from uuid import UUID + +from sqlalchemy import ForeignKey, inspect +from sqlalchemy.orm import Mapped, mapped_column, relationship +from sqlalchemy.sql import func + +from core.db.models import Base +from core.ui.base import UserInput as UserInputData + +if TYPE_CHECKING: + from core.db.models import Branch, ProjectState + + +class UserInput(Base): + __tablename__ = "user_inputs" + + # ID and parent FKs + id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True) + branch_id: Mapped[UUID] = mapped_column(ForeignKey("branches.id", ondelete="CASCADE")) + project_state_id: Mapped[Optional[UUID]] = mapped_column(ForeignKey("project_states.id", ondelete="SET NULL")) + + # Attributes + created_at: Mapped[datetime] = mapped_column(server_default=func.now()) + question: Mapped[str] = mapped_column() + answer_text: Mapped[Optional[str]] = mapped_column() + answer_button: Mapped[Optional[str]] = mapped_column() + cancelled: Mapped[bool] = mapped_column() + + # Relationships + branch: Mapped["Branch"] = relationship(back_populates="user_inputs", lazy="raise") + project_state: Mapped["ProjectState"] = relationship(back_populates="user_inputs", lazy="raise") + + @classmethod + def from_user_input(cls, project_state: "ProjectState", question: str, user_input: UserInputData) -> "UserInput": + """ + Store the user input in the database. + + Note this just creates the UserInput object. It is committed to the + database only when the DB session itself is comitted. + + :param project_state: Project state to associate the request log with. + :param question: Question the user was asked. + :param user_input: User input. + :return: Newly created User input in the database. + """ + session = inspect(project_state).async_session + + obj = cls( + project_state=project_state, + branch=project_state.branch, + question=question, + answer_text=user_input.text, + answer_button=user_input.button, + cancelled=user_input.cancelled, + ) + session.add(obj) + return obj diff --git a/core/db/session.py b/core/db/session.py new file mode 100644 index 00000000..f55460d0 --- /dev/null +++ b/core/db/session.py @@ -0,0 +1,75 @@ +from sqlalchemy import event +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + +from core.config import DBConfig +from core.log import get_logger + +log = get_logger(__name__) + + +class SessionManager: + """ + Async-aware context manager for database session. + + Usage: + + >>> config = DBConfig(url="sqlite+aiosqlite:///test.db") + >>> async with DBSession(config) as session: + ... # Do something with the session + """ + + def __init__(self, config: DBConfig): + """ + Initialize the session manager with the given configuration. + + :param config: Database configuration. + """ + self.config = config + self.engine = create_async_engine( + self.config.url, echo=config.debug_sql, echo_pool="debug" if config.debug_sql else None + ) + self.SessionClass = async_sessionmaker(self.engine, expire_on_commit=False) + self.session = None + self.recursion_depth = 0 + + event.listen(self.engine.sync_engine, "connect", self._on_connect) + + def _on_connect(self, dbapi_connection, _): + """Connection event handler""" + log.debug(f"Connected to database {self.config.url}") + + if self.config.url.startswith("sqlite"): + # Note that SQLite uses NullPool by default, meaning every session creates a + # database "connection". This is fine and preferred for SQLite because + # it's a local file. PostgreSQL or other database use a real connection pool + # by default. + dbapi_connection.execute("pragma foreign_keys=on") + + async def start(self) -> AsyncSession: + if self.session is not None: + self.recursion_depth += 1 + log.warning(f"Re-entering database session (depth: {self.recursion_depth}), potential bug", stack_info=True) + return self.session + + self.session = self.SessionClass() + return self.session + + async def close(self): + if self.session is None: + log.warning("Closing database session that was never opened", stack_info=True) + return + if self.recursion_depth > 0: + self.recursion_depth -= 1 + return + + await self.session.close() + self.session = None + + async def __aenter__(self) -> AsyncSession: + return await self.start() + + async def __aexit__(self, exc_type, exc_val, exc_tb): + return await self.close() + + +__all__ = ["SessionManager"] diff --git a/core/db/setup.py b/core/db/setup.py new file mode 100644 index 00000000..357a10fa --- /dev/null +++ b/core/db/setup.py @@ -0,0 +1,49 @@ +from os.path import dirname, join + +from alembic import command +from alembic.config import Config + +from core.config import DBConfig +from core.log import get_logger + +log = get_logger(__name__) + + +def _async_to_sync_db_scheme(url: str) -> str: + """ + Convert an async database URL to a synchronous one. + + This is needed because Alembic does not support async database + connections. + + :param url: Asynchronouse database URL. + :return: Synchronous database URL. + """ + if url.startswith("postgresql+asyncpg://"): + return url.replace("postgresql+asyncpg://", "postgresql://") + elif url.startswith("sqlite+aiosqlite://"): + return url.replace("sqlite+aiosqlite://", "sqlite://") + return url + + +def run_migrations(config: DBConfig): + """ + Run database migrations using Alembic. + + This needs to happen synchronously, before the asyncio + mainloop is started, and before any database access. + + :param config: Database configuration. + """ + url = _async_to_sync_db_scheme(config.url) + ini_location = join(dirname(__file__), "alembic.ini") + + log.debug(f"Running database migrations for {url} (config: {ini_location})") + + alembic_cfg = Config(ini_location) + alembic_cfg.set_main_option("sqlalchemy.url", url) + alembic_cfg.set_main_option("pythagora_runtime", "true") + command.upgrade(alembic_cfg, "head") + + +__all__ = ["run_migrations"] diff --git a/core/db/v0importer.py b/core/db/v0importer.py new file mode 100644 index 00000000..1ce37727 --- /dev/null +++ b/core/db/v0importer.py @@ -0,0 +1,274 @@ +from json import loads +from os.path import exists +from pathlib import Path +from uuid import UUID, uuid4 + +import aiosqlite + +from core.db.models import Branch, Project, ProjectState +from core.db.models.project_state import TaskStatus +from core.db.session import SessionManager +from core.disk.vfs import MemoryVFS +from core.log import get_logger +from core.state.state_manager import StateManager + +log = get_logger(__name__) + + +class ImporterStateManager(StateManager): + async def init_file_system(self, load_existing: bool) -> MemoryVFS: + """ + Initialize in-memory file system. + + We don't want to overwrite all the files on disk while importing + the legacy database, as this could overwrite new changes that the + user might have done in the meantime. Project loading will handle that. + """ + return MemoryVFS() + + +class LegacyDatabaseImporter: + def __init__(self, session_manager: SessionManager, dbpath: str): + self.session_manager = session_manager + self.state_manager = ImporterStateManager(self.session_manager, None) + self.dbpath = dbpath + self.conn = None + + if not exists(dbpath): + raise FileNotFoundError(f"File not found: {dbpath}") + + async def import_database(self): + try: + info = await self.load_legacy_database() + except Exception as err: # noqa + print(f"Failed to load legacy database {self.dbpath}: {err}") + return + n = await self.save_to_new_database(info) + print(f"Successfully imported {n} projects from {self.dbpath}") + + async def load_legacy_database(self): + async with aiosqlite.connect(self.dbpath) as conn: + self.conn = conn + is_valid = await self.verify_schema() + if not is_valid: + raise ValueError(f"Database {self.dbpath} doesn't look like a GPT-Pilot database") + + apps = await self.get_apps() + info = {} + for app_id in apps: + app_info = await self.get_app_info(app_id) + info[app_id] = { + "name": apps[app_id], + **app_info, + } + + return info + + async def verify_schema(self) -> bool: + tables = set() + async with self.conn.execute("select name from sqlite_master where type = 'table'") as cursor: + async for row in cursor: + tables.add(row[0]) + + return "app" in tables and "development_steps" in tables + + async def get_apps(self) -> dict[str, str]: + apps = {} + async with self.conn.execute("select id, name, status from app") as cursor: + async for id, name, status in cursor: + if status == "coding": + apps[id] = name + return apps + + async def get_app_info(self, app_id: str) -> dict: + app_info = { + "initial_prompt": None, + "architecture": None, + "tasks": [], + } + + async with self.conn.execute("select architecture from architecture where app_id = ?", (app_id,)) as cursor: + row = await cursor.fetchone() + if row: + app_info["architecture"] = loads(row[0]) + + async with self.conn.execute("select prompt from project_description where app_id = ?", (app_id,)) as cursor: + row = await cursor.fetchone() + if row: + app_info["initial_prompt"] = row[0] + + async with self.conn.execute( + "select id, prompt_path, prompt_data, messages, llm_response from development_steps " + "where app_id = ? order by created_at asc", + (app_id,), + ) as cursor: + async for row in cursor: + dev_step_id, prompt_path, prompt_data, messages, llm_response = row + if prompt_path == "development/task/breakdown.prompt": + task_info = await self.get_task_info(dev_step_id, prompt_data, llm_response) + app_info["tasks"].append(task_info) + + return app_info + + async def get_task_info(self, dev_step_id, prompt_data_json: str, llm_response: dict) -> dict: + prompt_data = loads(prompt_data_json) + current_feature = prompt_data.get("current_feature") + previous_features = prompt_data.get("previous_features") or [] + tasks = prompt_data["development_tasks"] + current_task_index = prompt_data["current_task_index"] + current_task = tasks[current_task_index] + instructions = llm_response + files = await self.get_task_files(dev_step_id) + return { + "current_feature": current_feature, + "previous_features": previous_features, + "tasks": tasks, + "current_task_index": current_task_index, + "current_task": current_task, + "instructions": instructions, + "files": files, + } + + async def get_task_files(self, dev_step_id: int): + files = {} + + async with self.conn.execute( + "select content, path, name, description from file_snapshot " + "inner join file on file_snapshot.file_id = file.id " + "where file_snapshot.development_step_id = ?", + (dev_step_id,), + ) as cursor: + async for row in cursor: + content, path, name, description = row + file_path = Path(path + "/" + name).as_posix() if path else name + try: + if isinstance(content, bytes): + content = content.decode("utf-8") + except: # noqa + # skip binary file + continue + files[file_path] = { + "description": description or None, + "content": content, + } + + return files + + async def save_to_new_database(self, info: dict) -> int: + """ + Save projects to the new database + + :param info: A dictionary with app_id as key and app info as value. + :return: Number of projects saved to the new database. + """ + async with self.session_manager as session: + projects = await Project.get_all_projects(session) + + for project in projects: + imported_app = info.pop(project.id.hex, None) + if imported_app: + log.info(f"Project {project.name} already exists in the new database, skipping") + + n = 0 + for app_id, app_info in info.items(): + await self.save_app(app_id, app_info) + n += 1 + return n + + async def save_app(self, app_id: str, app_info: dict): + log.info(f"Importing app {app_info['name']} (id={app_id}) ...") + + async with self.session_manager as session: + project = Project(id=UUID(app_id), name=app_info["name"]) + branch = Branch(project=project) + state = ProjectState.create_initial_state(branch) + + spec = state.specification + spec.description = app_info["initial_prompt"] + spec.architecture = app_info["architecture"]["architecture"] + spec.system_dependencies = app_info["architecture"]["system_dependencies"] + spec.package_dependencies = app_info["architecture"]["package_dependencies"] + spec.template = app_info["architecture"].get("template") + + session.add(project) + await session.commit() + + project = await self.state_manager.load_project(project_id=app_id) + + # It is much harder to import all tasks and keep features/tasks lists in sync, so + # we only support importing the latest task. + if app_info["tasks"]: + await self.save_latest_task(app_info["tasks"][-1]) + + # This just closes the session and removes the last (incomplete) state. + # Everything else should already be safely comitted. + await self.state_manager.rollback() + + async def save_latest_task(self, task: dict): + sm = self.state_manager + state = sm.current_state + + state.epics = [ + { + "id": uuid4().hex, + "name": "Initial Project", + "description": state.specification.description, + "summary": None, + "completed": bool(task["previous_features"]) or (task["current_feature"] is not None), + "complexity": "hard", + } + ] + + for i, feature in enumerate(task["previous_features"]): + state.epics += [ + { + "id": uuid4().hex, + "name": f"Feature #{i + 1}", + "description": feature["summary"], # FIXME: is this good enough + "summary": None, + "completed": True, + "complexity": "hard", + } + ] + + if task["current_feature"]: + state.epics = state.epics + [ + { + "id": uuid4().hex, + "name": f"Feature #{len(state.epics)}", + "description": task["current_feature"], + "summary": None, + "completed": False, + "complexity": "hard", + } + ] + + current_task_index = task["current_task_index"] + state.tasks = [ + { + "id": uuid4().hex, + "description": task_info["description"], + "instructions": None, + "status": TaskStatus.DONE if current_task_index > i else TaskStatus.TODO, + } + for i, task_info in enumerate(task["tasks"]) + ] + state.tasks[current_task_index]["instructions"] = task["instructions"] + await sm.current_session.commit() + + # Reload project at the initialized state to reinitialize the next state + await self.state_manager.load_project(project_id=state.branch.project.id, step_index=state.step_index) + + await self.save_task_files(task["files"]) + await self.state_manager.commit() + + async def save_task_files(self, files: dict): + for path, file_info in files.items(): + await self.state_manager.save_file( + path, + file_info["content"], + metadata={ + "description": file_info["description"], + "references": [], + }, + ) diff --git a/core/disk/__init__.py b/core/disk/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/core/disk/ignore.py b/core/disk/ignore.py new file mode 100644 index 00000000..ea25d944 --- /dev/null +++ b/core/disk/ignore.py @@ -0,0 +1,125 @@ +import os.path +from fnmatch import fnmatch +from typing import Optional + + +class IgnoreMatcher: + """ + A class to match paths against a list of ignore patterns or + file attributes (size, type). + """ + + def __init__( + self, + root_path: str, + ignore_paths: list[str], + *, + ignore_size_threshold: Optional[int] = None, + ): + """ + Initialize the IgnoreMatcher object. + + Ignore paths are matched agains the file name and the full path, + and may include shell-like wildcards ("*" for any number of characters, + "?" for a single character). Paths are normalized, so "/" works on both + Unix and Windows, and Windows matching is case insensitive. + + :param root_path: Root path to use when checking files on disk. + :param ignore_paths: List of patterns to ignore. + :param ignore_size_threshold: Files larger than this size will be ignored. + """ + self.root_path = root_path + self.ignore_paths = ignore_paths + self.ignore_size_threshold = ignore_size_threshold + + def ignore(self, path: str) -> bool: + """ + Check if the given path matches any of the ignore patterns. + + :param path: (Relative) path to the file or directory to check + :return: True if the path matches any of the ignore patterns, False otherwise + """ + + full_path = os.path.normpath(os.path.join(self.root_path, path)) + + if self._is_in_ignore_list(path): + return True + + if self._is_large_file(full_path): + return True + + # Binary files are always ignored + if self._is_binary(full_path): + return True + + return False + + def _is_in_ignore_list(self, path: str) -> bool: + """ + Check if the given path matches any of the ignore patterns. + + Both the (relative) file path and the file (base) name are matched. + + :param path: The path to the file or directory to check + :return: True if the path matches any of the ignore patterns, False otherwise. + """ + name = os.path.basename(path) + for pattern in self.ignore_paths: + if fnmatch(name, pattern) or fnmatch(path, pattern): + return True + return False + + def _is_large_file(self, full_path: str) -> bool: + """ + Check if the given file is larger than the threshold. + + This also returns True if the file doesn't or is not a regular file (eg. + it's a symlink), since we want to ignore those kinds of files as well. + + :param path: Full path to the file to check. + :return: True if the file is larger than the threshold, False otherwise. + """ + if self.ignore_size_threshold is None: + return False + + # We don't handle directories here + if os.path.isdir(full_path): + return False + + if not os.path.isfile(full_path): + return True + + try: + return bool(os.path.getsize(full_path) > self.ignore_size_threshold) + except: # noqa + return True + + def _is_binary(self, full_path: str) -> bool: + """ + Check if the given file is binary and should be ignored. + + This also returns True if the file doesn't or is not a regular file (eg. + it's a symlink), or can't be opened, since we want to ignore those too. + + :param path: Full path to the file to check. + :return: True if the file should be ignored, False otherwise. + """ + + # We don't handle directories here + if os.path.isdir(full_path): + return False + + if not os.path.isfile(full_path): + return True + + try: + with open(full_path, "r", encoding="utf-8") as f: + f.read(128 * 1024) + return False + except: # noqa + # If we can't open the file for any reason (eg. PermissionError), it's + # best to ignore it anyway + return True + + +__all__ = ["IgnoreMatcher"] diff --git a/core/disk/vfs.py b/core/disk/vfs.py new file mode 100644 index 00000000..a35b0611 --- /dev/null +++ b/core/disk/vfs.py @@ -0,0 +1,188 @@ +import os +import os.path +from hashlib import sha1 +from pathlib import Path + +from core.disk.ignore import IgnoreMatcher +from core.log import get_logger + +log = get_logger(__name__) + + +class VirtualFileSystem: + def save(self, path: str, content: str): + """ + Save content to a file. Use for both new and updated files. + + :param path: Path to the file, relative to project root. + :param content: Content to save. + """ + raise NotImplementedError() + + def read(self, path: str) -> str: + """ + Read file contents. + + :param path: Path to the file, relative to project root. + :return: File contents. + """ + raise NotImplementedError() + + def remove(self, path: str): + """ + Remove a file. + + If file doesn't exist or is a directory, or if the file is ignored, + do nothing. + + :param path: Path to the file, relative to project root. + """ + raise NotImplementedError() + + def get_full_path(self, path: str) -> str: + """ + Get the full path to a file. + + This should be used to check the full path of the file on whichever + file system it locally is stored. For example, getting a full path + to a file and then passing it to an external program via run_command + should work. + + :param path: Path to the file, relative to project root. + :return: Full path to the file. + """ + raise NotImplementedError() + + def _filter_by_prefix(self, file_list: list[str], prefix: str) -> list[str]: + # We use "/" internally on all platforms, including win32 + if not prefix.endswith("/"): + prefix = prefix + "/" + return [f for f in file_list if f.startswith(prefix)] + + def _get_file_list(self) -> list[str]: + raise NotImplementedError() + + def list(self, prefix: str = None) -> list[str]: + """ + Return a list of files in the project. + + File paths are relative to the project root. + + :param prefix: Optional prefix to filter files for. + :return: List of file paths. + """ + retval = sorted(self._get_file_list()) + if prefix: + retval = self._filter_by_prefix(retval, prefix) + return retval + + def hash(self, path: str) -> str: + content = self.read(path) + return self.hash_string(content) + + @staticmethod + def hash_string(content: str) -> str: + return sha1(content.encode("utf-8")).hexdigest() + + +class MemoryVFS(VirtualFileSystem): + files: dict[str, str] + + def __init__(self): + self.files = {} + + def save(self, path: str, content: str): + self.files[path] = content + + def read(self, path: str) -> str: + try: + return self.files[path] + except KeyError: + raise ValueError(f"File not found: {path}") + + def remove(self, path: str): + if path in self.files: + del self.files[path] + + def get_full_path(self, path: str) -> str: + # We use "/" internally on all platforms, including win32 + return "/" + path + + def _get_file_list(self) -> list[str]: + return self.files.keys() + + +class LocalDiskVFS(VirtualFileSystem): + def __init__( + self, + root: str, + create: bool = True, + allow_existing: bool = True, + ignore_matcher: IgnoreMatcher = None, + ): + if not os.path.isdir(root): + if create: + os.makedirs(root) + else: + raise ValueError(f"Root directory does not exist: {root}") + else: + if not allow_existing: + raise FileExistsError(f"Root directory already exists: {root}") + + if ignore_matcher is None: + ignore_matcher = IgnoreMatcher(root, []) + + self.root = root + self.ignore_matcher = ignore_matcher + + def get_full_path(self, path: str) -> str: + return os.path.abspath(os.path.normpath(os.path.join(self.root, path))) + + def save(self, path: str, content: str): + full_path = self.get_full_path(path) + os.makedirs(os.path.dirname(full_path), exist_ok=True) + with open(full_path, "w", encoding="utf-8") as f: + f.write(content) + log.debug(f"Saved file {path} ({len(content)} bytes) to {full_path}") + + def read(self, path: str) -> str: + full_path = self.get_full_path(path) + if not os.path.isfile(full_path): + raise ValueError(f"File not found: {path}") + + # TODO: do we want error handling here? + with open(full_path, "r", encoding="utf-8") as f: + return f.read() + + def remove(self, path: str): + if self.ignore_matcher.ignore(path): + return + + full_path = self.get_full_path(path) + if os.path.isfile(full_path): + try: + os.remove(full_path) + log.debug(f"Removed file {path} from {full_path}") + except Exception as err: # noqa + log.error(f"Failed to remove file {path}: {err}", exc_info=True) + + def _get_file_list(self) -> list[str]: + files = [] + for dpath, dirnames, filenames in os.walk(self.root): + # Modify in place to prevent recursing into ignored directories + dirnames[:] = [ + d + for d in dirnames + if not self.ignore_matcher.ignore(os.path.relpath(os.path.join(dpath, d), self.root)) + ] + + for filename in filenames: + path = os.path.relpath(os.path.join(dpath, filename), self.root) + if not self.ignore_matcher.ignore(path): + # We use "/" internally on all platforms, including win32 + files.append(Path(path).as_posix()) + + return files + + +__all__ = ["VirtualFileSystem", "MemoryVFS", "LocalDiskVFS"] diff --git a/core/docker/__init__.py b/core/docker/__init__.py new file mode 100644 index 00000000..7b522b91 --- /dev/null +++ b/core/docker/__init__.py @@ -0,0 +1,12 @@ +from .docker_client import DockerClient, DockerClientError + +__all__ = ['DockerClient', 'DockerClientError'] + +""" +Docker client module for interacting with Docker containers and images. +""" + +from .docker_client import DockerClient + +__all__ = ['DockerClient'] + diff --git a/core/docker/docker_client.py b/core/docker/docker_client.py new file mode 100644 index 00000000..140b8320 --- /dev/null +++ b/core/docker/docker_client.py @@ -0,0 +1,243 @@ +import asyncio +import logging +from typing import Dict, List, Optional, Union, Any + +import docker +from docker.errors import DockerException, ImageNotFound, NotFound, APIError + + +class DockerClientError(Exception): + """Base exception for Docker client errors.""" + pass + + +class DockerClient: + """ + Asynchronous wrapper for Docker SDK for Python. + Provides methods for container management, image operations, and other Docker tasks. + Supports async context manager protocol (async with). + """ + + def __init__( + self, + base_url: Optional[str] = None, + version: Optional[str] = None, + timeout: int = 60, + tls: bool = False, + ssl_version: Optional[str] = None, + logger: Optional[logging.Logger] = None, + ): + """ + Initialize Docker client with connection parameters. + + Args: + base_url: URL to the Docker server (e.g., unix:///var/run/docker.sock or tcp://127.0.0.1:1234) + version: API version to use + timeout: Request timeout + tls: Use TLS + ssl_version: SSL version to use + logger: Logger instance + """ + self.logger = logger or logging.getLogger(__name__) + + try: + self.client = docker.APIClient( + base_url=base_url, + version=version, + timeout=timeout, + tls=tls, + ssl_version=ssl_version, + ) + self.high_level_client = docker.from_env() + self.logger.info("Docker client initialized successfully") + except DockerException as e: + self.logger.error(f"Failed to initialize Docker client: {str(e)}") + raise DockerClientError(f"Failed to initialize Docker client: {str(e)}") + + async def __aenter__(self): + """ + Async context manager entry. + Returns: + self: The DockerClient instance + """ + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + """ + Async context manager exit. Closes the client and releases resources. + Args: + exc_type: Exception type if an exception was raised in the context + exc_val: Exception value if an exception was raised in the context + exc_tb: Exception traceback if an exception was raised in the context + """ + await self.close() + + async def ping(self) -> bool: + """Check Docker daemon connectivity.""" + try: + result = await self._run_async(self.client.ping) + return result == {"message": "OK"} + except DockerException as e: + self.logger.error(f"Docker ping failed: {str(e)}") + return False + + async def get_info(self) -> Dict[str, Any]: + """Get Docker system information.""" + try: + return await self._run_async(self.client.info) + except DockerException as e: + self.logger.error(f"Failed to get Docker info: {str(e)}") + raise DockerClientError(f"Failed to get Docker info: {str(e)}") + + # Container operations + async def create_container( + self, + image: str, + command: Optional[Union[str, List[str]]] = None, + name: Optional[str] = None, + environment: Optional[Dict[str, str]] = None, + volumes: Optional[Dict[str, Dict[str, str]]] = None, + ports: Optional[Dict[str, Union[int, List[int]]]] = None, + detach: bool = True, + **kwargs, + ) -> Dict[str, Any]: + """ + Create a new container. + + Args: + image: Image to use + command: Command to run + name: Container name + environment: Environment variables as dict + volumes: Volumes to mount + ports: Ports to expose + detach: Run container in the background + **kwargs: Additional parameters to pass to the Docker API + + Returns: + Dictionary with container information + """ + try: + config = { + "image": image, + "command": command, + "name": name, + "detach": detach, + "environment": environment, + "volumes": volumes, + "ports": ports, + **kwargs, + } + + # Remove None values + config = {k: v for k, v in config.items() if v is not None} + + self.logger.info(f"Creating container from image {image}") + return await self._run_async(self.client.create_container, **config) + except DockerException as e: + self.logger.error(f"Failed to create container: {str(e)}") + raise DockerClientError(f"Failed to create container: {str(e)}") + + async def start_container(self, container_id: str) -> None: + """Start a container.""" + try: + self.logger.info(f"Starting container {container_id}") + await self._run_async(self.client.start, container_id) + except DockerException as e: + self.logger.error(f"Failed to start container {container_id}: {str(e)}") + raise DockerClientError(f"Failed to start container {container_id}: {str(e)}") + + async def stop_container(self, container_id: str, timeout: int = 10) -> None: + """Stop a container with timeout.""" + try: + self.logger.info(f"Stopping container {container_id}") + await self._run_async(self.client.stop, container_id, timeout=timeout) + except NotFound: + self.logger.warning(f"Container {container_id} not found") + except DockerException as e: + self.logger.error(f"Failed to stop container {container_id}: {str(e)}") + raise DockerClientError(f"Failed to stop container {container_id}: {str(e)}") + + async def remove_container(self, container_id: str, force: bool = False) -> None: + """Remove a container.""" + try: + self.logger.info(f"Removing container {container_id}") + await self._run_async(self.client.remove_container, container_id, force=force) + except NotFound: + self.logger.warning(f"Container {container_id} not found") + except DockerException as e: + self.logger.error(f"Failed to remove container {container_id}: {str(e)}") + raise DockerClientError(f"Failed to remove container {container_id}: {str(e)}") + + async def list_containers(self, all: bool = False) -> List[Dict[str, Any]]: + """List containers.""" + try: + return await self._run_async(self.client.containers, all=all) + except DockerException as e: + self.logger.error(f"Failed to list containers: {str(e)}") + raise DockerClientError(f"Failed to list containers: {str(e)}") + + async def container_logs( + self, container_id: str, stdout: bool = True, stderr: bool = True, tail: str = "all" + ) -> str: + """Get container logs.""" + try: + logs = await self._run_async( + self.client.logs, container_id, stdout=stdout, stderr=stderr, tail=tail, stream=False + ) + return logs.decode("utf-8") if isinstance(logs, bytes) else logs + except NotFound: + self.logger.warning(f"Container {container_id} not found") + return "" + except DockerException as e: + self.logger.error(f"Failed to get logs for container {container_id}: {str(e)}") + raise DockerClientError(f"Failed to get logs for container {container_id}: {str(e)}") + + # Image operations + async def pull_image(self, repository: str, tag: str = "latest") -> None: + """Pull an image from a registry.""" + try: + self.logger.info(f"Pulling image {repository}:{tag}") + await self._run_async(self.client.pull, repository, tag=tag) + except DockerException as e: + self.logger.error(f"Failed to pull image {repository}:{tag}: {str(e)}") + raise DockerClientError(f"Failed to pull image {repository}:{tag}: {str(e)}") + + async def list_images(self) -> List[Dict[str, Any]]: + """List images.""" + try: + return await self._run_async(self.client.images) + except DockerException as e: + self.logger.error(f"Failed to list images: {str(e)}") + raise DockerClientError(f"Failed to list images: {str(e)}") + + async def remove_image(self, image: str, force: bool = False) -> None: + """Remove an image.""" + try: + self.logger.info(f"Removing image {image}") + await self._run_async(self.client.remove_image, image, force=force) + except ImageNotFound: + self.logger.warning(f"Image {image} not found") + except DockerException as e: + self.logger.error(f"Failed to remove image {image}: {str(e)}") + raise DockerClientError(f"Failed to remove image {image}: {str(e)}") + + # Utility methods + async def _run_async(self, func, *args, **kwargs): + """Run a synchronous Docker SDK function in a separate thread.""" + loop = asyncio.get_event_loop() + return await loop.run_in_executor( + None, lambda: func(*args, **kwargs) + ) + + async def close(self) -> None: + """Close all connections and resources. Called automatically when exiting async with block.""" + try: + if hasattr(self, 'client') and self.client: + await self._run_async(self.client.close) + if hasattr(self, 'high_level_client') and self.high_level_client: + await self._run_async(self.high_level_client.close) + self.logger.info("Docker client closed") + except Exception as e: + self.logger.error(f"Error closing Docker client: {str(e)}") + diff --git a/core/llm/__init__.py b/core/llm/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/core/llm/anthropic_client.py b/core/llm/anthropic_client.py new file mode 100644 index 00000000..796766dd --- /dev/null +++ b/core/llm/anthropic_client.py @@ -0,0 +1,154 @@ +import asyncio +import datetime +import zoneinfo +from typing import Optional, Tuple + +from anthropic import AsyncAnthropic, RateLimitError +from httpx import Timeout + +from core.config import LLMProvider +from core.llm.convo import Convo +from core.log import get_logger + +from .base import BaseLLMClient + +log = get_logger(__name__) + +# Maximum number of tokens supported by Anthropic Claude 3 +MAX_TOKENS = 4096 +MAX_TOKENS_SONNET = 8192 + + +class CustomAssertionError(Exception): + pass + + +class AnthropicClient(BaseLLMClient): + provider = LLMProvider.ANTHROPIC + + def _init_client(self): + self.client = AsyncAnthropic( + api_key=self.config.api_key, + base_url=self.config.base_url, + timeout=Timeout( + max(self.config.connect_timeout, self.config.read_timeout), + connect=self.config.connect_timeout, + read=self.config.read_timeout, + ), + ) + self.stream_handler = self.stream_handler + + def _adapt_messages(self, convo: Convo) -> list[dict[str, str]]: + """ + Adapt the conversation messages to the format expected by the Anthropic Claude model. + + Claude only recognizes "user" and "assistant" roles, and requires them to be switched + for each message (i.e. no consecutive messages from the same role). + + :param convo: Conversation to adapt. + :return: Adapted conversation messages. + """ + messages = [] + for msg in convo.messages: + if msg["role"] == "function": + raise ValueError("Anthropic Claude doesn't support function calling") + + role = "user" if msg["role"] in ["user", "system"] else "assistant" + if messages and messages[-1]["role"] == role: + messages[-1]["content"] += "\n\n" + msg["content"] + else: + messages.append( + { + "role": role, + "content": msg["content"], + } + ) + return messages + + async def _make_request( + self, convo: Convo, temperature: Optional[float] = None, json_mode: bool = False, retry_count: int = 1 + ) -> Tuple[str, int, int]: + async def single_attempt() -> Tuple[str, int, int]: + messages = self._adapt_messages(convo) + completion_kwargs = { + "max_tokens": MAX_TOKENS, + "model": self.config.model, + "messages": messages, + "temperature": self.config.temperature if temperature is None else temperature, + } + + if "trybricks" in self.config.base_url: + completion_kwargs["extra_headers"] = {"x-request-timeout": f"{int(float(self.config.read_timeout))}s"} + + if "bedrock/anthropic" in self.config.base_url: + completion_kwargs["extra_headers"] = {"anthropic-version": "bedrock-2023-05-31"} + + if "sonnet" in self.config.model: + completion_kwargs["max_tokens"] = MAX_TOKENS_SONNET + + if json_mode: + completion_kwargs["response_format"] = {"type": "json_object"} + + response = [] + async with self.client.messages.stream(**completion_kwargs) as stream: + async for content in stream.text_stream: + response.append(content) + if self.stream_handler: + await self.stream_handler(content) + + try: + final_message = await stream.get_final_message() + final_message.content # Access content to verify it exists + except AssertionError: + log.debug("Anthropic package AssertionError") + raise CustomAssertionError("No final message received.") + + response_str = "".join(response) + + # Tell the stream handler we're done + if self.stream_handler: + await self.stream_handler(None) + + return response_str, final_message.usage.input_tokens, final_message.usage.output_tokens + + for attempt in range(retry_count + 1): + try: + return await single_attempt() + except CustomAssertionError as e: + if attempt == retry_count: # If this was our last attempt + raise CustomAssertionError(f"Request failed after {retry_count + 1} attempts: {str(e)}") + # Add a small delay before retrying + await asyncio.sleep(1) + continue + + def rate_limit_sleep(self, err: RateLimitError) -> Optional[datetime.timedelta]: + """ + Anthropic rate limits docs: + https://docs.anthropic.com/en/api/rate-limits#response-headers + Limit reset times are in RFC 3339 format. + + """ + headers = err.response.headers + if "anthropic-ratelimit-tokens-remaining" not in headers: + return None + + remaining_tokens = headers["anthropic-ratelimit-tokens-remaining"] + if remaining_tokens == 0: + relevant_dt = headers["anthropic-ratelimit-tokens-reset"] + else: + relevant_dt = headers["anthropic-ratelimit-requests-reset"] + + try: + reset_time = datetime.datetime.fromisoformat(relevant_dt) + except ValueError: + return datetime.timedelta(seconds=5) + + try: + now = datetime.datetime.now(tz=zoneinfo.ZoneInfo("UTC")) + except zoneinfo.ZoneInfoNotFoundError: + now = datetime.datetime.now(tz=datetime.timezone.utc) + + return reset_time - now + + +__all__ = ["AnthropicClient"] diff --git a/core/llm/azure_client.py b/core/llm/azure_client.py new file mode 100644 index 00000000..300bbe45 --- /dev/null +++ b/core/llm/azure_client.py @@ -0,0 +1,29 @@ +from httpx import Timeout +from openai import AsyncAzureOpenAI + +from core.config import LLMProvider +from core.llm.openai_client import OpenAIClient +from core.log import get_logger + +log = get_logger(__name__) + + +class AzureClient(OpenAIClient): + provider = LLMProvider.AZURE + stream_options = None + + def _init_client(self): + azure_deployment = self.config.extra.get("azure_deployment") + api_version = self.config.extra.get("api_version") + + self.client = AsyncAzureOpenAI( + api_key=self.config.api_key, + azure_endpoint=self.config.base_url, + azure_deployment=azure_deployment, + api_version=api_version, + timeout=Timeout( + max(self.config.connect_timeout, self.config.read_timeout), + connect=self.config.connect_timeout, + read=self.config.read_timeout, + ), + ) diff --git a/core/llm/base.py b/core/llm/base.py new file mode 100644 index 00000000..4e141ea0 --- /dev/null +++ b/core/llm/base.py @@ -0,0 +1,366 @@ +import asyncio +import datetime +import json +from enum import Enum +from time import time +from typing import Any, Callable, Optional, Tuple + +import httpx + +from core.config import LLMConfig, LLMProvider +from core.llm.convo import Convo +from core.llm.request_log import LLMRequestLog, LLMRequestStatus +from core.log import get_logger + +log = get_logger(__name__) + + +class LLMError(str, Enum): + KEY_EXPIRED = "key_expired" + RATE_LIMITED = "rate_limited" + GENERIC_API_ERROR = "generic_api_error" + + +class APIError(Exception): + def __init__(self, message: str): + self.message = message + + +class BaseLLMClient: + """ + Base asynchronous streaming client for language models. + + Example usage: + + >>> async def stream_handler(content: str): + ... print(content) + ... + >>> def parser(content: str) -> dict: + ... return json.loads(content) + ... + >>> client_class = BaseClient.for_provider(provider) + >>> client = client_class(config, stream_handler=stream_handler) + >>> response, request_log = await client(convo, parser=parser) + """ + + provider: LLMProvider + + def __init__( + self, + config: LLMConfig, + *, + stream_handler: Optional[Callable] = None, + error_handler: Optional[Callable] = None, + ): + """ + Initialize the client with the given configuration. + + :param config: Configuration for the client. + :param stream_handler: Optional handler for streamed responses. + """ + self.config = config + self.stream_handler = stream_handler + self.error_handler = error_handler + self._init_client() + + def _init_client(self): + raise NotImplementedError() + + async def _make_request( + self, + convo: Convo, + temperature: Optional[float] = None, + json_mode: bool = False, + ) -> tuple[str, int, int]: + """ + Call the Anthropic Claude model with the given conversation. + + Low-level method that streams the response chunks. + Use `__call__` instead of this method. + + :param convo: Conversation to send to the LLM. + :param json_mode: If True, the response is expected to be JSON. + :return: Tuple containing the full response content, number of input tokens, and number of output tokens. + """ + raise NotImplementedError() + + async def _adapt_messages(self, convo: Convo) -> list[dict[str, str]]: + """ + Adapt the conversation messages to the format expected by the LLM. + + Claude only recognizes "user" and "assistant roles" + + :param convo: Conversation to adapt. + :return: Adapted conversation messages. + """ + messages = [] + for msg in convo.messages: + if msg.role == "function": + raise ValueError("Anthropic Claude doesn't support function calling") + + role = "user" if msg.role in ["user", "system"] else "assistant" + if messages and messages[-1]["role"] == role: + messages[-1]["content"] += "\n\n" + msg.content + else: + messages.append( + { + "role": role, + "content": msg.content, + } + ) + return messages + + async def __call__( + self, + convo: Convo, + *, + temperature: Optional[float] = None, + parser: Optional[Callable] = None, + max_retries: int = 3, + json_mode: bool = False, + ) -> Tuple[Any, LLMRequestLog]: + """ + Invoke the LLM with the given conversation. + + Stream handler, if provided, should be an async function + that takes a single argument, the response content (str). + It will be called for each response chunk. + + Parser, if provided, should be a function that takes the + response content (str) and returns the parsed response. + On parse error, the parser should raise a ValueError with + a descriptive error message that will be sent back to the LLM + to retry, up to max_retries. + + :param convo: Conversation to send to the LLM. + :param parser: Optional parser for the response. + :param max_retries: Maximum number of retries for parsing the response. + :param json_mode: If True, the response is expected to be JSON. + :return: Tuple of the (parsed) response and request log entry. + """ + import anthropic + import groq + import openai + + if temperature is None: + temperature = self.config.temperature + + convo = convo.fork() + request_log = LLMRequestLog( + provider=self.provider, + model=self.config.model, + temperature=temperature, + prompts=convo.prompt_log, + ) + + prompt_length_kb = len(json.dumps(convo.messages).encode("utf-8")) / 1024 + log.debug( + f"Calling {self.provider.value} model {self.config.model} (temp={temperature}), prompt length: {prompt_length_kb:.1f} KB" + ) + t0 = time() + + remaining_retries = max_retries + while True: + if remaining_retries == 0: + # We've run out of auto-retries + if request_log.error: + last_error_msg = f"Error connecting to the LLM: {request_log.error}" + else: + last_error_msg = "Error parsing LLM response" + + # If we can, ask the user if they want to keep retrying + if self.error_handler: + should_retry = await self.error_handler(LLMError.GENERIC_API_ERROR, message=last_error_msg) + if should_retry: + remaining_retries = max_retries + continue + + # They don't want to retry (or we can't ask them), raise the last error and stop Pythagora + raise APIError(last_error_msg) + + remaining_retries -= 1 + request_log.messages = convo.messages[:] + request_log.response = None + request_log.status = LLMRequestStatus.SUCCESS + request_log.error = None + response = None + + try: + response, prompt_tokens, completion_tokens = await self._make_request( + convo, + temperature=temperature, + json_mode=json_mode, + ) + except (openai.APIConnectionError, anthropic.APIConnectionError, groq.APIConnectionError) as err: + log.warning(f"API connection error: {err}", exc_info=True) + request_log.error = str(f"API connection error: {err}") + request_log.status = LLMRequestStatus.ERROR + continue + except httpx.ReadTimeout as err: + log.warning(f"Read timeout (set to {self.config.read_timeout}s): {err}", exc_info=True) + request_log.error = str(f"Read timeout: {err}") + request_log.status = LLMRequestStatus.ERROR + continue + except httpx.ReadError as err: + log.warning(f"Read error: {err}", exc_info=True) + request_log.error = str(f"Read error: {err}") + request_log.status = LLMRequestStatus.ERROR + continue + except (openai.RateLimitError, anthropic.RateLimitError, groq.RateLimitError) as err: + log.warning(f"Rate limit error: {err}", exc_info=True) + request_log.error = str(f"Rate limit error: {err}") + request_log.status = LLMRequestStatus.ERROR + wait_time = self.rate_limit_sleep(err) + if wait_time: + message = f"We've hit {self.config.provider.value} rate limit. Sleeping for {wait_time.seconds} seconds..." + if self.error_handler: + await self.error_handler(LLMError.RATE_LIMITED, message) + await asyncio.sleep(wait_time.seconds) + continue + else: + # RateLimitError that shouldn't be retried, eg. insufficient funds + err_msg = err.response.json().get("error", {}).get("message", "Rate limiting error.") + raise APIError(err_msg) from err + except (openai.NotFoundError, anthropic.NotFoundError, groq.NotFoundError) as err: + err_msg = err.response.json().get("error", {}).get("message", f"Model not found: {self.config.model}") + raise APIError(err_msg) from err + except (openai.AuthenticationError, anthropic.AuthenticationError, groq.AuthenticationError) as err: + log.warning(f"Key expired: {err}", exc_info=True) + err_msg = err.response.json().get("error", {}).get("message", "Incorrect API key") + if "[BricksLLM]" in err_msg: + # We only want to show the key expired message if it's from Bricks + if self.error_handler: + should_retry = await self.error_handler(LLMError.KEY_EXPIRED) + if should_retry: + continue + + raise APIError(err_msg) from err + except (openai.APIStatusError, anthropic.APIStatusError, groq.APIStatusError) as err: + # Token limit exceeded (in original gpt-pilot handled as + # TokenLimitError) is thrown as 400 (OpenAI, Anthropic) or 413 (Groq). + # All providers throw an exception that is caught here. + # OpenAI and Groq return a `code` field in the error JSON that lets + # us confirm that we've breached the token limit, but Anthropic doesn't, + # so we can't be certain that's the problem in Anthropic case. + # Here we try to detect that and tell the user what happened. + log.info(f"API status error: {err}") + try: + if hasattr(err, "response"): + if err.response.headers.get("Content-Type", "").startswith("application/json"): + err_code = err.response.json().get("error", {}).get("code", "") + else: + err_code = str(err.response.text) + elif isinstance(err, str): + err_code = err + else: + err_code = json.dumps(err) + except Exception as e: + err_code = f"Error parsing response: {str(e)}" + if err_code in ("request_too_large", "context_length_exceeded", "string_above_max_length"): + # Handle OpenAI and Groq token limit exceeded + # OpenAI will return `string_above_max_length` for prompts more than 1M characters + message = "".join( + [ + "We sent too large request to the LLM, resulting in an error. ", + "This is usually caused by including framework files in an LLM request. ", + "Here's how you can get Pythagora to ignore those extra files: ", + "https://bit.ly/faq-token-limit-error", + ] + ) + raise APIError(message) from err + + log.warning(f"API error: {err}", exc_info=True) + request_log.error = str(f"API error: {err}") + request_log.status = LLMRequestStatus.ERROR + continue + except (openai.APIError, anthropic.APIError, groq.APIError) as err: + # Generic LLM API error + # Make sure this handler is last in the chain as some of the above + # errors inherit from these `APIError` classes + log.warning(f"LLM API error {err}", exc_info=True) + request_log.error = f"LLM had an error processing our request: {err}" + request_log.status = LLMRequestStatus.ERROR + continue + + request_log.response = response + + request_log.prompt_tokens += prompt_tokens + request_log.completion_tokens += completion_tokens + if parser: + try: + response = parser(response) + break + except ValueError as err: + request_log.error = f"Error parsing response: {err}" + request_log.status = LLMRequestStatus.ERROR + log.debug(f"Error parsing LLM response: {err}, asking LLM to retry", exc_info=True) + convo.assistant(response) + convo.user(f"Error parsing response: {err}. Please output your response EXACTLY as requested.") + continue + else: + break + + t1 = time() + request_log.duration = t1 - t0 + + log.debug( + f"Total {self.provider.value} response time {request_log.duration:.2f}s, {request_log.prompt_tokens} prompt tokens, {request_log.completion_tokens} completion tokens used" + ) + + return response, request_log + + async def api_check(self) -> bool: + """ + Perform an LLM API check. + + :return: True if the check was successful, False otherwise. + """ + + convo = Convo() + msg = "This is a connection test. If you can see this, please respond only with 'START' and nothing else." + convo.user(msg) + resp, _log = await self(convo) + return bool(resp) + + @staticmethod + def for_provider(provider: LLMProvider) -> type["BaseLLMClient"]: + """ + Return LLM client for the specified provider. + + :param provider: Provider to return the client for. + :return: Client class for the specified provider. + """ + from .anthropic_client import AnthropicClient + from .azure_client import AzureClient + from .groq_client import GroqClient + from .openai_client import OpenAIClient + + if provider == LLMProvider.OPENAI: + return OpenAIClient + elif provider == LLMProvider.ANTHROPIC: + return AnthropicClient + elif provider == LLMProvider.GROQ: + return GroqClient + elif provider == LLMProvider.AZURE: + return AzureClient + else: + raise ValueError(f"Unsupported LLM provider: {provider.value}") + + def rate_limit_sleep(self, err: Exception) -> Optional[datetime.timedelta]: + """ + Return how long we need to sleep because of rate limiting. + + These are computed from the response headers that each LLM returns. + For details, check the implementation for the specific LLM. If there + are no rate limiting headers, we assume that the request should not + be retried and return None (this will be the case for insufficient + quota/funds in the account). + + :param err: RateLimitError that was raised by the LLM client. + :return: optional timedelta to wait before trying again + """ + + raise NotImplementedError() + + +__all__ = ["BaseLLMClient"] diff --git a/core/llm/convo.py b/core/llm/convo.py new file mode 100644 index 00000000..ae6af656 --- /dev/null +++ b/core/llm/convo.py @@ -0,0 +1,170 @@ +from copy import deepcopy +from typing import Any, Iterator, Optional + + +class Convo: + """ + A conversation between a user and a Large Language Model (LLM) assistant. + + Holds messages and an optional metadata log (list of dicts with + prompt information). + """ + + ROLES = ["system", "user", "assistant", "function"] + + messages: list[dict[str, str]] + prompt_log: list[dict[str, Any]] + + def __init__(self, content: Optional[str] = None): + """ + Initialize a new conversation. + + :param content: Initial system message (optional). + """ + self.messages = [] + self.prompt_log = [] + + if content is not None: + self.system(content) + + @staticmethod + def _dedent(text: str) -> str: + """ + Remove common leading whitespace from every line of text. + + :param text: Text to dedent. + :return: Dedented text. + """ + indent = len(text) + lines = text.splitlines() + for line in lines: + if line.strip(): + indent = min(indent, len(line) - len(line.lstrip())) + dedented_lines = [line[indent:].rstrip() for line in lines] + return "\n".join(line for line in dedented_lines) + + def add(self, role: str, content: str, name: Optional[str] = None) -> "Convo": + """ + Add a message to the conversation. + + In most cases, you should use the convenience methods instead. + + :param role: Role of the message (system, user, assistant, function). + :param content: Content of the message. + :param name: Name of the message sender (optional). + :return: The conv object. + """ + + if role not in self.ROLES: + raise ValueError(f"Unknown role: {role}") + if not content: + raise ValueError("Empty message content") + if not isinstance(content, str) and not isinstance(content, dict): + raise TypeError(f"Invalid message content: {type(content).__name__}") + + message = { + "role": role, + "content": self._dedent(content) if isinstance(content, str) else content, + } + if name is not None: + message["name"] = name + + self.messages.append(message) + return self + + def system(self, content: str, name: Optional[str] = None) -> "Convo": + """ + Add a system message to the conversation. + + System messages can use `name` for showing example conversations + between an example user and an example assistant. + + :param content: Content of the message. + :param name: Name of the message sender (optional). + :return: The convo object. + """ + return self.add("system", content, name) + + def user(self, content: str, name: Optional[str] = None) -> "Convo": + """ + Add a user message to the conversation. + + :param content: Content of the message. + :param name: User name (optional). + :return: The convo object. + """ + return self.add("user", content, name) + + def assistant(self, content: str, name: Optional[str] = None) -> "Convo": + """ + Add an assistant message to the conversation. + + :param content: Content of the message. + :param name: Assistant name (optional). + :return: The convo object. + """ + return self.add("assistant", content, name) + + def function(self, content: str, name: Optional[str] = None) -> "Convo": + """ + Add a function (tool) response to the conversation. + + :param content: Content of the message. + :param name: Function/tool name (optional). + :return: The convo object. + """ + return self.add("function", content, name) + + def fork(self) -> "Convo": + """ + Create an identical copy of the conversation. + + This performs a deep copy of all the message + contents, so you can safely modify both the + parent and the child conversation. + + :return: A copy of the conversation. + """ + child = Convo() + child.messages = deepcopy(self.messages) + child.prompt_log = deepcopy(self.prompt_log) + return child + + def after(self, parent: "Convo") -> "Convo": + """ + Create a chat with only messages after the last common + message (that appears in both parent conversation and + this one). + + :param parent: Parent conversation. + :return: A new conversation with only new messages. + """ + index = 0 + while index < min(len(self.messages), len(parent.messages)) and self.messages[index] == parent.messages[index]: + index += 1 + + child = Convo() + child.messages = [deepcopy(msg) for msg in self.messages[index:]] + return child + + def last(self) -> Optional[dict[str, str]]: + """ + Get the last message in the conversation. + + :return: The last message, or None if the conversation is empty. + """ + return self.messages[-1] if self.messages else None + + def __iter__(self) -> Iterator[dict[str, str]]: + """ + Iterate over the messages in the conversation. + + :return: An iterator over the messages. + """ + return iter(self.messages) + + def __repr__(self) -> str: + return f"" + + +__all__ = ["Convo"] diff --git a/core/llm/groq_client.py b/core/llm/groq_client.py new file mode 100644 index 00000000..2021abdc --- /dev/null +++ b/core/llm/groq_client.py @@ -0,0 +1,93 @@ +import datetime +from typing import Optional + +import tiktoken +from groq import AsyncGroq, RateLimitError +from httpx import Timeout + +from core.config import LLMProvider +from core.llm.base import BaseLLMClient +from core.llm.convo import Convo +from core.log import get_logger + +log = get_logger(__name__) +tokenizer = tiktoken.get_encoding("cl100k_base") + + +class GroqClient(BaseLLMClient): + provider = LLMProvider.GROQ + + def _init_client(self): + self.client = AsyncGroq( + api_key=self.config.api_key, + base_url=self.config.base_url, + timeout=Timeout( + max(self.config.connect_timeout, self.config.read_timeout), + connect=self.config.connect_timeout, + read=self.config.read_timeout, + ), + ) + + async def _make_request( + self, + convo: Convo, + temperature: Optional[float] = None, + json_mode: bool = False, + ) -> tuple[str, int, int]: + completion_kwargs = { + "model": self.config.model, + "messages": convo.messages, + "temperature": self.config.temperature if temperature is None else temperature, + "stream": True, + } + if json_mode: + completion_kwargs["response_format"] = {"type": "json_object"} + + stream = await self.client.chat.completions.create(**completion_kwargs) + response = [] + prompt_tokens = 0 + completion_tokens = 0 + + async for chunk in stream: + if not chunk.choices: + continue + + content = chunk.choices[0].delta.content + if not content: + continue + + response.append(content) + if self.stream_handler: + await self.stream_handler(content) + + response_str = "".join(response) + + # Tell the stream handler we're done + if self.stream_handler: + await self.stream_handler(None) + + if prompt_tokens == 0 and completion_tokens == 0: + # FIXME: Here we estimate Groq tokens using the same method as for OpenAI.... + # See https://cookbook.openai.com/examples/how_to_count_tokens_with_tiktoken + prompt_tokens = sum(3 + len(tokenizer.encode(msg["content"])) for msg in convo.messages) + completion_tokens = len(tokenizer.encode(response_str)) + + return response_str, prompt_tokens, completion_tokens + + def rate_limit_sleep(self, err: RateLimitError) -> Optional[datetime.timedelta]: + """ + Groq rate limits docs: https://console.groq.com/docs/rate-limits + + Groq includes `retry-after` header when 429 RateLimitError is + thrown, so we use that instead of calculating our own backoff time. + """ + + headers = err.response.headers + if "retry-after" not in headers: + return None + + retry_after = int(err.response.headers["retry-after"]) + return datetime.timedelta(seconds=retry_after) + + +__all__ = ["GroqClient"] diff --git a/core/llm/openai_client.py b/core/llm/openai_client.py new file mode 100644 index 00000000..1933ab54 --- /dev/null +++ b/core/llm/openai_client.py @@ -0,0 +1,120 @@ +import datetime +import re +from typing import Optional + +import tiktoken +from httpx import Timeout +from openai import AsyncOpenAI, RateLimitError + +from core.config import LLMProvider +from core.llm.base import BaseLLMClient +from core.llm.convo import Convo +from core.log import get_logger + +log = get_logger(__name__) +tokenizer = tiktoken.get_encoding("cl100k_base") + + +class OpenAIClient(BaseLLMClient): + provider = LLMProvider.OPENAI + stream_options = {"include_usage": True} + + def _init_client(self): + self.client = AsyncOpenAI( + api_key=self.config.api_key, + base_url=self.config.base_url, + timeout=Timeout( + max(self.config.connect_timeout, self.config.read_timeout), + connect=self.config.connect_timeout, + read=self.config.read_timeout, + ), + ) + + async def _make_request( + self, + convo: Convo, + temperature: Optional[float] = None, + json_mode: bool = False, + ) -> tuple[str, int, int]: + completion_kwargs = { + "model": self.config.model, + "messages": convo.messages, + "temperature": self.config.temperature if temperature is None else temperature, + "stream": True, + } + if self.stream_options: + completion_kwargs["stream_options"] = self.stream_options + + if json_mode: + completion_kwargs["response_format"] = {"type": "json_object"} + + stream = await self.client.chat.completions.create(**completion_kwargs) + response = [] + prompt_tokens = 0 + completion_tokens = 0 + + async for chunk in stream: + if chunk.usage: + prompt_tokens += chunk.usage.prompt_tokens + completion_tokens += chunk.usage.completion_tokens + + if not chunk.choices: + continue + + content = chunk.choices[0].delta.content + if not content: + continue + + response.append(content) + if self.stream_handler: + await self.stream_handler(content) + + response_str = "".join(response) + + # Tell the stream handler we're done + if self.stream_handler: + await self.stream_handler(None) + + if prompt_tokens == 0 and completion_tokens == 0: + # See https://cookbook.openai.com/examples/how_to_count_tokens_with_tiktoken + prompt_tokens = sum(3 + len(tokenizer.encode(msg["content"])) for msg in convo.messages) + completion_tokens = len(tokenizer.encode(response_str)) + log.warning( + "OpenAI response did not include token counts, estimating with tiktoken: " + f"{prompt_tokens} input tokens, {completion_tokens} output tokens" + ) + + return response_str, prompt_tokens, completion_tokens + + def rate_limit_sleep(self, err: RateLimitError) -> Optional[datetime.timedelta]: + """ + OpenAI rate limits docs: + https://platform.openai.com/docs/guides/rate-limits/error-mitigation + Limit reset times are in "2h32m54s" format. + """ + + headers = err.response.headers + if "x-ratelimit-remaining-tokens" not in headers: + return None + + remaining_tokens = headers["x-ratelimit-remaining-tokens"] + time_regex = r"(?:(\d+)h)?(?:(\d+)m)?(?:(\d+)s)?" + if remaining_tokens == 0: + match = re.search(time_regex, headers["x-ratelimit-reset-tokens"]) + else: + match = re.search(time_regex, headers["x-ratelimit-reset-requests"]) + + if match: + hours = int(match.group(1)) if match.group(1) else 0 + minutes = int(match.group(2)) if match.group(2) else 0 + seconds = int(match.group(3)) if match.group(3) else 0 + total_seconds = hours * 3600 + minutes * 60 + seconds + else: + # Not sure how this would happen, we would have to get a RateLimitError, + # but nothing (or invalid entry) in the `reset` field. Using a sane default. + total_seconds = 5 + + return datetime.timedelta(seconds=total_seconds) + + +__all__ = ["OpenAIClient"] diff --git a/core/llm/parser.py b/core/llm/parser.py new file mode 100644 index 00000000..e2330690 --- /dev/null +++ b/core/llm/parser.py @@ -0,0 +1,226 @@ +import json +import re +from enum import Enum +from typing import List, Optional, Union + +from pydantic import BaseModel, ValidationError, create_model + + +class CodeBlock(BaseModel): + description: str + content: str + + +class ParsedBlocks(BaseModel): + original_response: str + blocks: List[CodeBlock] + + +class DescriptiveCodeBlockParser: + """ + Parse Markdown code blocks with their descriptions from a string. + Returns both the original response and structured data about each block. + + Each block entry contains: + - description: The text line immediately preceding the code block + - content: The actual content of the code block + + Example usage: + >>> parser = DescriptiveCodeBlockParser() + >>> text = '''file: next.config.js + ... ```js + ... module.exports = { + ... reactStrictMode: true, + ... }; + ... ```''' + >>> result = parser(text) + >>> assert result.blocks[0].description == "file: next.config.js" + """ + + def __init__(self): + self.pattern = re.compile(r"^(.*?)\n```([a-z0-9]+\n)?(.*?)^```\s*", re.DOTALL | re.MULTILINE) + + def __call__(self, text: str) -> ParsedBlocks: + # Store original response + original_response = text.strip() + + # Find all blocks with their preceding text + blocks = [] + for match in self.pattern.finditer(text): + description = match.group(1).strip() + content = match.group(3).strip() + + # Only add block if we have both description and content + if description and content: + blocks.append(CodeBlock(description=description, content=content)) + + return ParsedBlocks(original_response=original_response, blocks=blocks) + + +class MultiCodeBlockParser: + """ + Parse multiple Markdown code blocks from a string. + + Expects zero or more blocks, and ignores any text + outside of the code blocks. + + Example usage: + + >>> parser = MultiCodeBlockParser() + >>> text = ''' + ... text outside block + ... + ... ```python + ... first block + ... ``` + ... some text between blocks + ... ```js + ... more + ... code + ... ``` + ... some text after blocks + ''' + >>> assert parser(text) == ["first block", "more\ncode"] + + If no code blocks are found, an empty list is returned: + """ + + def __init__(self): + self.pattern = re.compile(r"^```([a-z0-9]+\n)?(.*?)^```\s*", re.DOTALL | re.MULTILINE) + + def __call__(self, text: str) -> list[str]: + blocks = [] + for block in self.pattern.findall(text): + blocks.append(block[1].strip()) + return blocks + + +class CodeBlockParser(MultiCodeBlockParser): + """ + Parse a Markdown code block from a string. + + Expects exactly one code block, and ignores + any text before or after it. + + Usage: + >>> parser = CodeBlockParser() + >>> text = "text\n```py\ncodeblock\n'''\nmore text" + >>> assert parser(text) == "codeblock" + + This is a special case of MultiCodeBlockParser, + checking that there's exactly one block. + """ + + def __call__(self, text: str) -> str: + blocks = super().__call__(text) + # FIXME: if there are more than 1 code block, this means the output actually contains ```, + # so re-parse this with that in mind + if len(blocks) != 1: + raise ValueError(f"Expected a single code block, got {len(blocks)}") + return blocks[0] + + +class OptionalCodeBlockParser: + def __call__(self, text: str) -> str: + text = text.strip() + if text.startswith("```") and text.endswith("\n```"): + # Remove the first and last line. Note the first line may include syntax + # highlighting, so we can't just remove the first 3 characters. + text = "\n".join(text.splitlines()[1:-1]).strip() + elif "\n" not in text and text.startswith("`") and text.endswith("`"): + # Single-line code blocks are wrapped in single backticks + text = text[1:-1] + return text + + +class JSONParser: + def __init__(self, spec: Optional[BaseModel] = None, strict: bool = True): + self.spec = spec + self.strict = strict or (spec is not None) + self.original_response = None + + @property + def schema(self): + return self.spec.model_json_schema() if self.spec else None + + @staticmethod + def errors_to_markdown(errors: list) -> str: + error_txt = [] + for error in errors: + loc = ".".join(str(loc) for loc in error["loc"]) + etype = error["type"] + msg = error["msg"] + error_txt.append(f"- `{loc}`: {etype} ({msg})") + return "\n".join(error_txt) + + def __call__(self, text: str) -> Union[BaseModel, dict, None]: + self.original_response = text.strip() # Store the original text + text = self.original_response + if text.startswith("```"): + try: + text = CodeBlockParser()(text) + except ValueError: + if self.strict: + raise + else: + return None + + try: + data = json.loads(text.strip()) + except json.JSONDecodeError as e: + if self.strict: + raise ValueError(f"JSON is not valid: {e}") from e + else: + return None + if self.spec is None: + return data + + try: + model = self.spec(**data) + except ValidationError as err: + errtxt = self.errors_to_markdown(err.errors()) + raise ValueError(f"Invalid JSON format:\n{errtxt}") from err + except Exception as err: + raise ValueError(f"Error parsing JSON: {err}") from err + + # Create a new model that includes the original model fields and the original text + ExtendedModel = create_model( + f"Extended{self.spec.__name__}", + original_response=(str, ...), + **{field_name: (field.annotation, field.default) for field_name, field in self.spec.__fields__.items()}, + ) + + # Instantiate the extended model + extended_model = ExtendedModel(original_response=self.original_response, **model.dict()) + + return extended_model + + +class EnumParser: + def __init__(self, spec: Enum, ignore_case: bool = True): + self.spec = spec + self.ignore_case = ignore_case + + def __call__(self, text: str) -> Enum: + text = text.strip() + if self.ignore_case: + text = text.lower() + try: + return self.spec(text) + except ValueError as e: + options = ", ".join([str(v) for v in self.spec]) + raise ValueError(f"Invalid option '{text}'; valid options: {options}") from e + + +class StringParser: + def __call__(self, text: str) -> str: + # Strip any leading and trailing whitespace + text = text.strip() + + # Check and remove quotes at the start and end if they match + if text.startswith(("'", '"')) and text.endswith(("'", '"')) and len(text) > 1: + # Remove the first and last character if they are both quotes + if text[0] == text[-1]: + text = text[1:-1] + + return text diff --git a/core/llm/prompt.py b/core/llm/prompt.py new file mode 100644 index 00000000..03f938e0 --- /dev/null +++ b/core/llm/prompt.py @@ -0,0 +1,48 @@ +from os.path import isdir +from typing import Any, Optional + +from jinja2 import BaseLoader, Environment, FileSystemLoader, StrictUndefined, TemplateNotFound + + +class FormatTemplate: + def __call__(self, template: str, **kwargs: dict[str, Any]) -> str: + return template.format(**kwargs) + + +class BaseJinjaTemplate: + def __init__(self, loader: Optional[BaseLoader]): + self.env = Environment( + loader=loader, + autoescape=False, + lstrip_blocks=True, + trim_blocks=True, + keep_trailing_newline=True, + undefined=StrictUndefined, + ) + + +class JinjaStringTemplate(BaseJinjaTemplate): + def __init__(self): + super().__init__(None) + + def __call__(self, template: str, **kwargs: dict[str, Any]) -> str: + tpl = self.env.from_string(template) + return tpl.render(**kwargs) + + +class JinjaFileTemplate(BaseJinjaTemplate): + def __init__(self, template_dirs: list[str]): + for td in template_dirs: + if not isdir(td): + raise ValueError(f"Template directory does not exist: {td}") + super().__init__(FileSystemLoader(template_dirs)) + + def __call__(self, template: str, **kwargs: dict[str, Any]) -> str: + try: + tpl = self.env.get_template(template) + except TemplateNotFound as err: + raise ValueError(f"Template not found: {template}") from err + return tpl.render(**kwargs) + + +__all__ = ["FormatTemplate", "JinjaStringTemplate", "JinjaFileTemplate"] diff --git a/core/llm/request_log.py b/core/llm/request_log.py new file mode 100644 index 00000000..2500e9ef --- /dev/null +++ b/core/llm/request_log.py @@ -0,0 +1,30 @@ +from datetime import datetime +from enum import Enum +from typing import Any + +from pydantic import BaseModel, Field + +from core.config import LLMProvider + + +class LLMRequestStatus(str, Enum): + SUCCESS = "success" + ERROR = "error" + + +class LLMRequestLog(BaseModel): + provider: LLMProvider + model: str + temperature: float + messages: list[dict[str, str]] = Field(default_factory=list) + prompts: list[dict[str, Any]] = Field(default_factory=list) + response: str = "" + prompt_tokens: int = 0 + completion_tokens: int = 0 + started_at: datetime = Field(default_factory=datetime.now) + duration: float = 0.0 + status: LLMRequestStatus = LLMRequestStatus.SUCCESS + error: str = "" + + +__all__ = ["LLMRequestLog", "LLMRequestStatus"] diff --git a/core/log/__init__.py b/core/log/__init__.py new file mode 100644 index 00000000..67eb2ffc --- /dev/null +++ b/core/log/__init__.py @@ -0,0 +1,50 @@ +from logging import FileHandler, Formatter, Logger, StreamHandler, getLogger + +from core.config import LogConfig + + +def setup(config: LogConfig, force: bool = False): + """ + Set up logging based on the current configuration. + + The method is idempotent unless `force` is set to True, + in which case it will reconfigure the logging. + """ + + root = getLogger() + logger = getLogger("core") + # Only clear/remove existing log handlers if we're forcing a new setup + if not force and (root.handlers or logger.handlers): + return + + while force and root.handlers: + root.removeHandler(root.handlers[0]) + + while force and logger.handlers: + logger.removeHandler(logger.handlers[0]) + + level = config.level + formatter = Formatter(config.format) + + if config.output: + handler = FileHandler(config.output, encoding="utf-8") + else: + handler = StreamHandler() + + handler.setFormatter(formatter) + handler.setLevel(level) + + logger.setLevel(level) + logger.addHandler(handler) + + +def get_logger(name) -> Logger: + """ + Get log function for a given (module) name + + :return: Logger instance + """ + return getLogger(name) + + +__all__ = ["setup", "get_logger"] diff --git a/core/proc/__init__.py b/core/proc/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/core/proc/exec_log.py b/core/proc/exec_log.py new file mode 100644 index 00000000..bfe7a073 --- /dev/null +++ b/core/proc/exec_log.py @@ -0,0 +1,21 @@ +from datetime import datetime, timezone +from typing import Optional + +from pydantic import BaseModel, Field + + +class ExecLog(BaseModel): + started_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + duration: float = Field(description="The duration of the command/process run in seconds") + cmd: str = Field(description="The full command (as executed in the shell)") + cwd: str = Field(description="The working directory for the command (relative to project root)") + env: dict = Field(description="The environment variables for the command") + timeout: Optional[float] = Field(description="The command timeout in seconds (or None if no timeout)") + status_code: Optional[int] = Field(description="The command return code, or None if there was a timeout") + stdout: str = Field(description="The command standard output") + stderr: str = Field(description="The command standard error") + analysis: str = Field(description="The result analysis as performed by the LLM") + success: bool = Field(description="Whether the command was successful") + + +__all__ = ["ExecLog"] diff --git a/core/proc/process_manager.py b/core/proc/process_manager.py new file mode 100644 index 00000000..c103d010 --- /dev/null +++ b/core/proc/process_manager.py @@ -0,0 +1,282 @@ +import asyncio +import signal +import sys +import time +from copy import deepcopy +from dataclasses import dataclass +from os import environ +from os.path import abspath, join +from typing import Callable, Optional +from uuid import UUID, uuid4 + +import psutil + +from core.log import get_logger + +log = get_logger(__name__) + +NONBLOCK_READ_TIMEOUT = 0.01 +BUSY_WAIT_INTERVAL = 0.1 +WATCHER_IDLE_INTERVAL = 1.0 +MAX_COMMAND_TIMEOUT = 180 + + +@dataclass +class LocalProcess: + id: UUID + cmd: str + cwd: str + env: dict[str, str] + stdout: str + stderr: str + _process: asyncio.subprocess.Process + + def __hash__(self) -> int: + return hash(self.id) + + @staticmethod + async def start( + cmd: str, + *, + cwd: str = ".", + env: dict[str, str], + bg: bool = False, + ) -> "LocalProcess": + log.debug(f"Starting process: {cmd} (cwd={cwd})") + _process = await asyncio.create_subprocess_shell( + cmd, + cwd=cwd, + env=env, + start_new_session=bg, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + if bg: + _process.stdin.close() + + return LocalProcess( + id=uuid4(), + cmd=cmd, + cwd=cwd, + env=env, + stdout="", + stderr="", + _process=_process, + ) + + async def wait(self, timeout: Optional[float] = None) -> int: + try: + future = self._process.wait() + if timeout: + future = asyncio.wait_for(future, timeout) + retcode = await future + except asyncio.TimeoutError: + log.debug(f"Process {self.cmd} still running after {timeout}s, terminating") + await self.terminate() + # FIXME: this may still hang if we don't manage to kill the process. + retcode = await self._process.wait() + + return retcode + + @staticmethod + async def _nonblock_read(reader: asyncio.StreamReader, timeout: float) -> str: + """ + Reads data from a stream reader without blocking (for long). + + This wraps the read in a (short) timeout to avoid blocking the event loop for too long. + + :param reader: Async stream reader to read from. + :param timeout: Timeout for the read operation (should not be too long). + :return: Data read from the stream reader, or empty string. + """ + buffer = "" + while True: + try: + data = await asyncio.wait_for(reader.read(1), timeout) + if not data: + return buffer + buffer += data.decode("utf-8", errors="ignore") + except asyncio.TimeoutError: + return buffer + + async def read_output(self, timeout: float = NONBLOCK_READ_TIMEOUT) -> tuple[str, str]: + new_stdout = await self._nonblock_read(self._process.stdout, timeout) + new_stderr = await self._nonblock_read(self._process.stderr, timeout) + self.stdout += new_stdout + self.stderr += new_stderr + return (new_stdout, new_stderr) + + async def _terminate_process_tree(self, signal: int): + # This is a recursive function that terminates the entire process tree + # of the current process. It first terminates all child processes, then + # terminates itself. + shell_process = psutil.Process(self._process.pid) + processes = shell_process.children(recursive=True) + processes.append(shell_process) + for proc in processes: + try: + proc.send_signal(signal) + except psutil.NoSuchProcess: + pass + + psutil.wait_procs(processes, timeout=1) + + async def terminate(self, kill: bool = True): + if kill and sys.platform != "win32": + await self._terminate_process_tree(signal.SIGKILL) + else: + # Windows doesn't have SIGKILL + await self._terminate_process_tree(signal.SIGTERM) + + @property + def is_running(self) -> bool: + try: + return psutil.Process(self._process.pid).is_running() + except psutil.NoSuchProcess: + return False + + @property + def pid(self) -> int: + return self._process.pid + + +class ProcessManager: + def __init__( + self, + *, + root_dir: str, + env: Optional[dict[str, str]] = None, + output_handler: Optional[Callable] = None, + exit_handler: Optional[Callable] = None, + ): + if env is None: + env = deepcopy(environ) + self.processes: dict[UUID, LocalProcess] = {} + self.default_env = env + self.root_dir = root_dir + self.watcher_should_run = True + self.watcher_task = asyncio.create_task(self.watcher()) + self.output_handler = output_handler + self.exit_handler = exit_handler + + async def stop_watcher(self): + """ + Stop the process watcher. + + This should only be done when the ProcessManager is no longer needed. + """ + if not self.watcher_should_run: + raise ValueError("Process watcher is not running") + + self.watcher_should_run = False + await self.watcher_task + + async def watcher(self): + """ + Watch over the processes and manage their output and lifecycle. + + This is a separate coroutine running independently of the caller + coroutine. + """ + # IDs of processes whos output has been fully read after they finished + complete_processes = set() + + while self.watcher_should_run: + procs = [p for p in self.processes.values() if p.id not in complete_processes] + if len(procs) == 0: + await asyncio.sleep(WATCHER_IDLE_INTERVAL) + continue + + for process in procs: + out, err = await process.read_output() + if self.output_handler and (out or err): + await self.output_handler(out, err) + + if not process.is_running: + # We're not removing the complete process from the self.processes + # list to give time to the rest of the system to read its outputs + complete_processes.add(process.id) + if self.exit_handler: + await self.exit_handler(process) + + # Sleep a bit to avoid busy-waiting + await asyncio.sleep(BUSY_WAIT_INTERVAL) + + async def start_process( + self, + cmd: str, + *, + cwd: str = ".", + env: Optional[dict[str, str]] = None, + bg: bool = True, + ) -> LocalProcess: + env = {**self.default_env, **(env or {})} + abs_cwd = abspath(join(self.root_dir, cwd)) + process = await LocalProcess.start(cmd, cwd=abs_cwd, env=env, bg=bg) + if bg: + self.processes[process.id] = process + return process + + async def run_command( + self, + cmd: str, + *, + cwd: str = ".", + env: Optional[dict[str, str]] = None, + timeout: float = MAX_COMMAND_TIMEOUT, + show_output: Optional[bool] = True, + ) -> tuple[Optional[int], str, str]: + """ + Run command and wait for it to finish. + + Status code is an integer representing the process exit code, or + None if the process timed out and was terminated. + + :param cmd: Command to run. + :param cwd: Working directory. + :param env: Environment variables. + :param timeout: Timeout in seconds. + :param show_output: Show output in the ui. + :return: Tuple of (status code, stdout, stderr). + """ + timeout = min(timeout, MAX_COMMAND_TIMEOUT) + terminated = False + process = await self.start_process(cmd, cwd=cwd, env=env, bg=False) + + t0 = time.time() + while process.is_running and (time.time() - t0) < timeout: + out, err = await process.read_output(BUSY_WAIT_INTERVAL) + if self.output_handler and (out or err) and show_output: + await self.output_handler(out, err) + + if process.is_running: + log.debug(f"Process {cmd} still running after {timeout}s, terminating") + await process.terminate() + terminated = True + else: + await process.wait() + + out, err = await process.read_output() + if self.output_handler and (out or err) and show_output: + await self.output_handler(out, err) + + if terminated: + status_code = None + else: + status_code = process._process.returncode or 0 + + return (status_code, process.stdout, process.stderr) + + def list_running_processes(self): + return [p for p in self.processes.values() if p.is_running] + + async def terminate_process(self, process_id: UUID) -> tuple[str, str]: + if process_id not in self.processes: + raise ValueError(f"Process {process_id} not found") + + process = self.processes[process_id] + await process.terminate(kill=False) + del self.processes[process_id] + + return (process.stdout, process.stderr) diff --git a/core/prompts/architect/configure_template.prompt b/core/prompts/architect/configure_template.prompt new file mode 100644 index 00000000..38715118 --- /dev/null +++ b/core/prompts/architect/configure_template.prompt @@ -0,0 +1,10 @@ +You're starting a new software project. The specification provided by the client is: + +``` +{{ project_description }} +``` + +Based on the specification, we've decided to use the following project scaffolding/template: {{ project_template.description }}. + +To start, we need to specify options for the project template: +{{ project_template.options_description }} diff --git a/core/prompts/architect/select_templates.prompt b/core/prompts/architect/select_templates.prompt new file mode 100644 index 00000000..e195fbcf --- /dev/null +++ b/core/prompts/architect/select_templates.prompt @@ -0,0 +1,33 @@ +You're designing the architecture and technical specifications for a new project. + +To speed up the project development, you need to consider if you should use a project template or start from scratch. If you decide to use a template, you should choose the one that best fits the project requirements. + +Here is a high level description of "{{ state.branch.project.name }}": +``` +{{ state.specification.description }} +``` + +You have an option to use project templates that implement standard boilerplate/scaffolding so you can start faster and be more productive. To be considered, a template must be compatible with the project requirements: +* if the project description has specific technology requirements, don't consider templates that choose different tech (eg. a different framework or library) +* to be considered, the template must use compatible technologies and implement a useful subset of required functionality + +If no project templates are a good match, don't pick any! It's better to start from scratch than to use a template that is not a good fit for the project (for example, don't use a react frontend if a different framework or plain html/css is required) and then spend time reworking it to fit the requirements. If you do choose to pick a template, choose the one that's the best match for this project. + +Here are the available project templates: + +{% for template in templates.values() %} +### {{ template.name }} ({{ template.stack }}) + +{{ template.description }} + +Contains: +{{ template.summary }} + +{% endfor %} +Output your response in a valid JSON format like in this example: +```json +{ + "architecture": "Detailed description of the architecture of the application", + "template": "foo" // or null if you decide not to use a project template +} +``` diff --git a/core/prompts/architect/system.prompt b/core/prompts/architect/system.prompt new file mode 100644 index 00000000..d7d6ae13 --- /dev/null +++ b/core/prompts/architect/system.prompt @@ -0,0 +1,2 @@ +You are a world class software architect. +You focus on creating architecture for Minimum Viable Product versions of apps developed as fast as possible with as many ready-made technologies as possible. \ No newline at end of file diff --git a/core/prompts/architect/technologies.prompt b/core/prompts/architect/technologies.prompt new file mode 100644 index 00000000..73cbe288 --- /dev/null +++ b/core/prompts/architect/technologies.prompt @@ -0,0 +1,52 @@ +You're designing the architecture and technical specifications for a new project. + +If the project requirements call out for specific technology, use that. Otherwise, if working on a web app, prefer Node.js for the backend (with Express if a web server is needed, and MongoDB if a database is needed), and Bootstrap for the front-end. You MUST NOT use Docker, Kubernetes, microservices and single-page app frameworks like React, Next.js, Angular, Vue or Svelte unless the project details explicitly require it. + +Here is a high level description of "{{ state.branch.project.name }}": +``` +{{ state.specification.description }} +``` + +Here is a short description of the project architecture: +{{ architecture }} + +Based on these details, think step by step and choose technologies to use in building it. + +1. First, list any system dependencies that should be installed on the system prior to start of development. For each system depedency, output a {{ os }} command to check whether it's installed. +2. Then, list any other 3rd party packages or libraries that will be used (that will be installed later using packager a package manager in the project repository/environment). +3. Finally, list the folder structure of the project, including any key files that should be included. + +*IMPORTANT*: You must follow these rules while creating your project: + +* You must only list *system* dependencies, ie. the ones that need to be installed (typically as admin) to set up the programming language, database, etc. Any packages that will need to be installed via language/platform-specific package managers are *not* system dependencies. +* If there are several popular options (such as Nginx or Apache for web server), pick one that would be more suitable for the app in question. +* DO NOT include text editors, IDEs, shells, OpenSSL, CLI tools such as git, AWS, or Stripe clients, or other utilities in your list. only direct dependencies required to build and run the project. +* If a dependency (such as database) has a cloud alternative or can be installed on another computer (ie. isn't required on this computer), you must mark it as `required_locally: false` + +Output only your response in JSON format like in this example, without other commentary: +```json +{ + "system_dependencies": [ + { + "name": "Node.js", + "description": "JavaScript runtime for building apps. This is required to be able to run the app you're building.", + "test": "node --version", + "required_locally": true + }, + { + "name": "MongoDB", + "description": "NoSQL database. If you don't want to install MongoDB locally, you can use a cloud version such as MongoDB Atlas.", + "test": "mongosh --version", + "required_locally": false + }, + ... + ], + "package_dependencies": [ + { + "name": "express", + "description": "Express web server for Node" + }, + ... + ] +} +``` diff --git a/core/prompts/bug-hunter/ask_a_question.prompt b/core/prompts/bug-hunter/ask_a_question.prompt new file mode 100644 index 00000000..348ee3e5 --- /dev/null +++ b/core/prompts/bug-hunter/ask_a_question.prompt @@ -0,0 +1,4 @@ +The developer wants to ask you a question. Here is the question: +{{question}} + +Please answer and refer to all the files in the repository and everything we've talked about so far but do not form your answer in any way that was asked for before, just answer the question as if you're talking to a colleague. diff --git a/core/prompts/bug-hunter/bug_found_or_add_logs.prompt b/core/prompts/bug-hunter/bug_found_or_add_logs.prompt new file mode 100644 index 00000000..6fd38704 --- /dev/null +++ b/core/prompts/bug-hunter/bug_found_or_add_logs.prompt @@ -0,0 +1,6 @@ +We are working on a solving a technical problem in a codebase and here is a conclusion from a team member: +--- TEAM_MEMBER_CONCLUSION --- +{{ hunt_conclusion }} +--- END_OF_TEAM_MEMBER_CONCLUSION --- + +Please tell me if the conclusion from the team member is to add more logs around the code or if the conclusion is that there are all information needed to fix the issue. diff --git a/core/prompts/bug-hunter/data_about_logs.prompt b/core/prompts/bug-hunter/data_about_logs.prompt new file mode 100644 index 00000000..630b2fc8 --- /dev/null +++ b/core/prompts/bug-hunter/data_about_logs.prompt @@ -0,0 +1,7 @@ +Tell me the most important logs that are relevant for this issue. For each log, tell me the the following: + 1. line in the code (eg. `print(...)`, `console.log(...)`, etc.) that generated the log + 2. what file is the log in (eg. `index.js`, `routes/users.js`, etc. - make sure to put the entire path like listed above) + 2. the current output of that log (make sure not to put the entire log output but maximum 5-10 lines of the output) + 3. the expected output of that log (also make sure to put maximum of 5-10 lines of the output) + 4. should the log be different from the current output or are the current and expected output the same + 5. a brief explanation of why the output is incorrect and what should be different here (use maximum 2-3 sentences) diff --git a/core/prompts/bug-hunter/get_bug_reproduction_instructions.prompt b/core/prompts/bug-hunter/get_bug_reproduction_instructions.prompt new file mode 100644 index 00000000..fcef8cf9 --- /dev/null +++ b/core/prompts/bug-hunter/get_bug_reproduction_instructions.prompt @@ -0,0 +1,30 @@ +You are working on an app called "{{ state.branch.project.name }}" and you need to write code for the entire application. + +{% include "partials/project_details.prompt" %} + +{% if state.tasks and state.current_task %} +Development process of this app was split into smaller tasks. Here is the list of all tasks: +```{% for task in state.tasks %} +{{ loop.index }}. {{ task.description }} +{% endfor %} +``` + +You are currently working on, and have to focus only on, this task: +``` +{{ current_task.description }} +``` + +{% endif %} +A part of the app is already finished. +{% include "partials/files_list.prompt" %} + +{% include "partials/user_feedback.prompt" %} + +{% if next_solution_to_try is not none %} +Focus on solving this issue in the following way: +``` +{{ next_solution_to_try }} +``` +{% endif %} + +Based on this information, you need to tell me in 2-3 sentences how can I reproduce the issue that the user experienced. diff --git a/core/prompts/bug-hunter/instructions_from_human_hint.prompt b/core/prompts/bug-hunter/instructions_from_human_hint.prompt new file mode 100644 index 00000000..a2d95cad --- /dev/null +++ b/core/prompts/bug-hunter/instructions_from_human_hint.prompt @@ -0,0 +1,6 @@ +The human is sending you a hint about how to solve this bug. Here is what human said: +``` +{{ human_hint }} +``` + +Now, based on this hint, break down exactly what the problem is, what is the solution to this problem and how can we implement this solution so that the bug is fixed. diff --git a/core/prompts/bug-hunter/iteration.prompt b/core/prompts/bug-hunter/iteration.prompt new file mode 100644 index 00000000..c4b48f17 --- /dev/null +++ b/core/prompts/bug-hunter/iteration.prompt @@ -0,0 +1,45 @@ +You are working on an app called "{{ state.branch.project.name }}" and you need to write code for the entire application. + +{% include "partials/project_details.prompt" %} + +{% if state.tasks and state.current_task %} +Development process of this app was split into smaller tasks. Here is the list of all tasks: +```{% for task in state.tasks %} +{{ loop.index }}. {{ task.description }} +{% endfor %} +``` + +You are currently working on, and have to focus only on, this task: +``` +{{ current_task.description }} +``` + +{% endif %} +A part of the app is already finished. +{% include "partials/files_list.prompt" %} + +{% include "partials/user_feedback.prompt" %} + +{% if test_instructions %} +Here are the test instructions the user was following when the issue occurred: +``` +{% for step in test_instructions %} +Step #{{ loop.index }} +Action: {{ step.action }} +Expected result: {{ step.result }} +{% endfor %} +``` +{% endif %} + +{% if next_solution_to_try is not none %} +Focus on solving this issue in the following way: +``` +{{ next_solution_to_try }} +``` +{% endif %} + +Based on this information, you need to figure out where is the problem that the user described. You have 2 options - to tell me exactly where is the problem happening or to add more logs to better determine where is the problem. +If you think we should add more logs around the code to better understand the problem, tell me code snippets in which we should add the logs. If you think you know where the issue is, don't add any new logs but explain what log print tell point you to the problem, what the problem is, what is the solution to this problem and how the solution will fix the problem. What is your answer? + +**IMPORTANT** +If you want code to be written, write **ALL NEW CODE** that needs to be written. If you want to create a new file, write the entire content of that file and if you want to update an existing file, write the new code that needs to be written/updated. You cannot answer with "Ensure that...", "Make sure that...", etc. In these cases, explain how should the reader of your message ensure what you want them to ensure. In most cases, they will need to add some logs to ensure something in which case tell them where to add them. diff --git a/core/prompts/bug-hunter/log_data.prompt b/core/prompts/bug-hunter/log_data.prompt new file mode 100644 index 00000000..a497236b --- /dev/null +++ b/core/prompts/bug-hunter/log_data.prompt @@ -0,0 +1,25 @@ +{% if backend_logs and backend_logs|trim %} +Here are the logs we added to the backend: +``` +{{ backend_logs }} +``` +{% endif %}{% if frontend_logs and frontend_logs|trim %} +Here are the logs we added to the frontend: +``` +{{ frontend_logs }} +``` +{% endif %}{% if user_feedback and user_feedback|trim %} +Finally, here is a hint from a human who tested the app: +``` +{{ user_feedback }} +``` +When you're thinking about what to do next, take into the account human's feedback. +{% endif %}{% if fix_attempted %} +The problem wasn't solved with the last changes. You have 2 options - to tell me exactly where is the problem happening or to add more logs to better determine where is the problem. If you think we should add more logs around the code to better understand the problem, tell me code snippets in which we should add the logs. If you think you know where the issue is, don't add any new logs but explain what log print tell point you to the problem, what the problem is, what is the solution to this problem and how the solution will fix the problem. What is your answer? Make sure not to repeat mistakes from before that didn't work. +{% endif %} +{% if not (backend_logs and backend_logs|trim) and + not (frontend_logs and frontend_logs|trim) and + not (user_feedback and user_feedback|trim) and + not fix_attempted %} +Human didn't supply any data +{% endif %} diff --git a/core/prompts/bug-hunter/problem_explanation.prompt b/core/prompts/bug-hunter/problem_explanation.prompt new file mode 100644 index 00000000..3064d1d0 --- /dev/null +++ b/core/prompts/bug-hunter/problem_explanation.prompt @@ -0,0 +1,4 @@ +This also didn't help to solve the issue so we can conclude that you are unable to solve this problem yourself so I got a human here who will help you out. + +First, focus on the problem you're facing and explain it to the human. Explain what is the issue that you're working in and what should the human try to do to solve this problem. Is there anything the human can look at that you don't have access to - a database, API response, etc.? If there is something for the human to look at, specify exactly how can the human obtain this information. Keep the answer short and to the point. + diff --git a/core/prompts/bug-hunter/system.prompt b/core/prompts/bug-hunter/system.prompt new file mode 100644 index 00000000..e69de29b diff --git a/core/prompts/bug-hunter/tell_me_more.prompt b/core/prompts/bug-hunter/tell_me_more.prompt new file mode 100644 index 00000000..5a25ff0f --- /dev/null +++ b/core/prompts/bug-hunter/tell_me_more.prompt @@ -0,0 +1 @@ +Please tell me more about the problem we're working on and don't repeat things you said before but tell me something I don't know. diff --git a/core/prompts/code-monkey/breakdown.prompt b/core/prompts/code-monkey/breakdown.prompt new file mode 100644 index 00000000..e5bf532f --- /dev/null +++ b/core/prompts/code-monkey/breakdown.prompt @@ -0,0 +1,2 @@ +{# This is the same template as for Developer's breakdown because Code Monkey is reusing it in a conversation #} +{% extends "developer/breakdown.prompt" %} diff --git a/core/prompts/code-monkey/describe_file.prompt b/core/prompts/code-monkey/describe_file.prompt new file mode 100644 index 00000000..6a8c1d37 --- /dev/null +++ b/core/prompts/code-monkey/describe_file.prompt @@ -0,0 +1,26 @@ +Your task is to explain the functionality implemented by a particular source code file. + +Given a file path and file contents, your output should contain: + +* a detailed explanation of what the file is about; +* a list of all other files referenced (imported) from this file. note that some libraries, frameworks or libraries assume file extension and don't use it explicitly. For example, "import foo" in Python references "foo.py" without specifying the extension. In your response, use the complete file name including the implied extension (for example "foo.py", not just "foo"). + +Please analyze file `{{ path }}`, which contains the following content: +``` +{{ content }} +``` + +Output the result in a JSON format with the following structure, as in this example: + +Example: +{ + "summary": "Describe in detail the functionality being defined or implemented in this file. Be as detailed as possible", + "references": [ + "some/file.py", + "some/other/file.js" + ], +} + +**IMPORTANT** In references, only include references to files that are local to the project. Do not include standard libraries or well-known external dependencies. + +Your response must be a valid JSON document, following the example format. Do not add any extra explanation or commentary outside the JSON document. diff --git a/core/prompts/code-monkey/implement_changes.prompt b/core/prompts/code-monkey/implement_changes.prompt new file mode 100644 index 00000000..14f92d0b --- /dev/null +++ b/core/prompts/code-monkey/implement_changes.prompt @@ -0,0 +1,40 @@ +{% if file_content %} +You are working on a project and your job is to implement new code changes based on given instructions. +Now you have to implement ALL changes that are related to `{{ file_name }}` described in development instructions listed below. +Make sure you don't make any mistakes, especially ones that could affect rest of project. Your changes will be reviewed by very detailed reviewer. Because of that, it is extremely important that you are STRICTLY following ALL the following rules while implementing changes: +{% else %} +You are working on a project and your job is to create a new file `{{ file_name }}` based on given instructions. The file should be thoroughly described in the development instructions listed below. You need to follow the coding rules that will be listed below, read the development instructions and respond with the full contents of the file `{{ file_name }}`. +{% endif %} +{% include "partials/coding_rules.prompt" %} + +{% include "partials/user_feedback.prompt" %} + +Here are development instructions that were sent to you by a senior developer that you need to carefully follow. Focus only on the code changes for the file `{{ file_name }}`: +~~~START_OF_DEVELOPMENT_INSTRUCTIONS~~~ + +{{ instructions }} + +~~~END_OF_DEVELOPMENT_INSTRUCTIONS~~~ + +{% if rework_feedback is defined %} +You previously made changes to file `{{ file_name }}` but not all changes were accepted, and the reviewer provided feedback on the changes that you must rework: +{{ rework_feedback}} +Please update the file accordingly and output the full new version of the file. + +The reviewer accepted some of your changes, and the file now looks like this: +``` +{{ file_content }} +``` +{% elif file_content %} +Now, take a look at how `{{ file_name }}` looks like currently: +``` +{{ file_content }} +``` + +Ok, now, you have to follow the instructions about `{{ file_name }}` from the development instructions carefully. Reply **ONLY** with the full contents of the file `{{ file_name }}` and nothing else. Do not make any changes to the file that are not mentioned in the development instructions - you must **STRICTLY** follow the instructions. +{% else %} +You need to create a new file `{{ file_name }}` so respond **ONLY** with the full contents of that file from the development instructions that you read. +{% endif %} + +** IMPORTANT ** +Remember, you must **NOT** add anything in your response that is not strictly the code from the file. Do not start or end the response with an explanation or a comment - you must respond with only the code from the file because your response will be directly saved to a file and run. diff --git a/core/prompts/code-monkey/iteration.prompt b/core/prompts/code-monkey/iteration.prompt new file mode 100644 index 00000000..97d7d10c --- /dev/null +++ b/core/prompts/code-monkey/iteration.prompt @@ -0,0 +1 @@ +{% extends "troubleshooter/iteration.prompt" %} \ No newline at end of file diff --git a/core/prompts/code-monkey/review_changes.prompt b/core/prompts/code-monkey/review_changes.prompt new file mode 100644 index 00000000..9b665593 --- /dev/null +++ b/core/prompts/code-monkey/review_changes.prompt @@ -0,0 +1,29 @@ +A developer on your team has been working on the task described in previous message. Based on those instructions, the developer has made changes to file `{{ file_name }}`. + +Here is the original content of this file: +``` +{{ old_content }} +``` + +Here is the diff of the changes: + +{% for hunk in hunks %}## Hunk {{ loop.index }} +```diff +{{ hunk }} +``` +{% endfor %} + +As you can see, there {% if hunks|length == 1 %}is only one hunk in this diff, and it{% else %}are {{hunks|length}} hunks in this diff, and each{% endif %} starts with the `@@` header line. + +When reviewing the code changes, apply these principles to decide on each hunk: +- Apply: Approve and integrate the hunk into our core codebase if it accurately delivers the intended functionality or enhancement, aligning with our project objectives. This action confirms the change is beneficial and meets our quality standards. +- Ignore: Use this option sparingly, only when you're certain the entire hunk is incorrect or will introduce errors (logical, syntax, etc.) that could negatively impact the project. Ignoring means the hunk will be completely removed. This should be reserved for cases where the inclusion of the code is definitively more harmful than its absence. Emphasize careful consideration before choosing 'Ignore.' It's crucial for situations where the hunk's removal is the only option to prevent significant issues. Otherwise, 'Rework' might be the better choice to ensure the code's integrity and functionality. +- Rework: Suggest this option if the concept behind the change is valid and necessary but is implemented in a way that introduces problems. This indicates a need for a revision of the hunk to refine its integration without fully discarding the underlying idea. DO NOT suggest making changes to files other than the one you're reviewing. + +When deciding what should be done with the hunk you are currently reviewing, pick an option that most reviewers of your skill would choose. Your decisions have to be consistent. + +Keep in mind you're just reviewing current file. You don't need to consider if other files are created, dependent packages installed, etc. Focus only on reviewing the changes in this file based on the instructions in the previous message. + +Note that the developer may add, modify or delete logging (including `gpt_pilot_debugging_log`) or error handling that's not explicitly asked for, but is a part of good development practice. Unless these logging and error handling additions break something, your decision to apply, ignore or rework the hunk should not be based on this. Base your decision only on functional changes - comments or logging are less important. Importantly, don't ask for a rework just because of logging or error handling changes. Also, take into account this is a junior developer and while the approach they take may not be the best practice, if it's not *wrong*, let it pass. Ask for rework only if the change is clearly bad and would break something. + +The developer that wrote this is sometimes sloppy and has could have deleted some parts of the code that contain important functionality and should not be deleted. Pay special attention to that in your review. diff --git a/core/prompts/code-monkey/review_feedback.prompt b/core/prompts/code-monkey/review_feedback.prompt new file mode 100644 index 00000000..cfb252e9 --- /dev/null +++ b/core/prompts/code-monkey/review_feedback.prompt @@ -0,0 +1,17 @@ +Your changes have been reviewed. +{% if content != original_content %} +The reviewer approved and applied some of your changes, but requested you rework the others. + +Here's the file with the approved changes already applied: +``` +{{ content }} +``` + +Here's the reviewer's feedback: +{% else %} +The reviewer requested that you rework your changes, here's the feedback: +{% endif %} + +{{ rework_feedback }} + +Based on this feedback and the original instructions, think carefully, make the correct changes, and output the entire file again. Remember, Output ONLY the content for this file, without additional explanation, suggestions or notes. Your output MUST start with ``` and MUST end with ``` and include only the complete file contents. diff --git a/core/prompts/code-monkey/system.prompt b/core/prompts/code-monkey/system.prompt new file mode 100644 index 00000000..76253982 --- /dev/null +++ b/core/prompts/code-monkey/system.prompt @@ -0,0 +1,3 @@ +You are a world class full stack software developer. +You write modular, clean, maintainable, production-ready code. +Your job is to implement tasks assigned by your tech lead. diff --git a/core/prompts/developer/breakdown.prompt b/core/prompts/developer/breakdown.prompt new file mode 100644 index 00000000..23bf770b --- /dev/null +++ b/core/prompts/developer/breakdown.prompt @@ -0,0 +1,64 @@ +You are working on an app called "{{ state.branch.project.name }}" and you are a primary developer who needs to write and maintain the code for this app.You are currently working on the implementation of one task that I will tell you below. Before that, here is the context of the app you're working on. Each section of the context starts with `~~SECTION_NAME~~` and ends with ~~END_OF_SECTION_NAME~~`. + +{% include "partials/project_details.prompt" %} +{% include "partials/features_list.prompt" %} +{% include "partials/files_list.prompt" %} + +{% include "partials/doc_snippets.prompt" %} + +{%- if state.epics|length == 1 %} +**IMPORTANT** +Remember, I created an empty folder where I will start writing files that you tell me and that are needed for this app. +{% endif %} +~~DEVELOPMENT_INSTRUCTIONS~~ +{% include "partials/relative_paths.prompt" %} + +DO NOT specify commands to create any folders or files, they will be created automatically - just specify the relative path to each file that needs to be written. + +{% include "partials/file_naming.prompt" %} +{% include "partials/execution_order.prompt" %} +{% include "partials/human_intervention_explanation.prompt" %} +{% include "partials/file_size_limit.prompt" %} +{% include "partials/breakdown_code_instructions.prompt" %} + +{% if state.has_frontend() %} +The entire backend API needs to be on /api/... routes! + +** IMPORTANT - Mocking API requests ** +Frontend side is making requests to the backend by calling functions that are defined in the folder client/api/. During the frontend implementation, API requests are mocked with dummy data that is defined in this folder and the API response data structure is defined in a comment above each API calling function. Whenever you need to implement an API endpoint, you must first find the function on the frontend that should call that API, remove the mocked data and make sure that the API call is properly done and that the response is parsed in a proper way. Whenever you do this, make sure to tell me explicitly which API calling function is being changed and what will be the response from the API. +Whenever you add an API request from the frontend, make sure to wrap the request in try/catch block and in the catch block, return `throw new Error(error?.response?.data?.error || error.message);` - in the place where the API request function is being called, show a toast message with an error. +{% endif %} + +** IMPORTANT - current implementation ** +Pay close attention to the currently implemented files, and DO NOT tell me to implementat something that is already implemented. Similarly, do not change the current implementation if you think it is working correctly. It is not necessary for you to change files - you can leave the files as they are and just tell me that they are correctly implemented. +~~END_OF_DEVELOPMENT_INSTRUCTIONS~~ + +~~DEVELOPMENT_PLAN~~ +We've broken the development of this {% if state.epics|length > 1 %}feature{% else %}app{% endif %} down to these tasks: +``` +{% for task in state.tasks %} +{{ loop.index }}. {{ task.description }}{% if task.get("status") == "done" %} (completed){% endif %} +{% endfor %} +``` +~~END_OF_DEVELOPMENT_PLAN~~ + +You are currently working on task #{{ current_task_index + 1 }} with the following description: +``` +{{ task.description }} +``` +{% if related_api_endpoints|length > 0 %} +In this task, you need to focus on implementing the following endpoints:{% for api in related_api_endpoints %}{{ "`" ~ api.endpoint ~ "`" }}{% if not loop.last %},{% endif %}{% endfor %} + + +{% endif %} +You must implement the backend API endpoints, remove the mocked that on the frontend side, and replace it with the real API request, implement the database model (if it's not implemented already), and implement the utility function (eg. 3rd party integration) that is needed for this endpoint. Whenever you need to create a utility function that uses a 3rd party integration, you **MUST** mock that function (create the function, return a mocked data, and specify the structure of the input and the output of the function in the comment above the function). + +{% if task.get('pre_breakdown_testing_instructions') is not none %} +Here is how this task should be tested: +``` +{{ task.pre_breakdown_testing_instructions }} +```{% endif %} + +{% if current_task_index != 0 %}All previous tasks are finished and you don't have to work on them.{% endif %} + +Now, start by writing up what needs to be implemented to get this task working. Think about how routes are set up, how are variables called, and other important things, and mention files by name and where should all new functionality be called from. Then, tell me all the code that needs to be written to implement ONLY this task and have it fully working and all commands that need to be run to implement this task. diff --git a/core/prompts/developer/filter_files.prompt b/core/prompts/developer/filter_files.prompt new file mode 100644 index 00000000..495ce414 --- /dev/null +++ b/core/prompts/developer/filter_files.prompt @@ -0,0 +1,40 @@ +We're starting work on a new task for a project we're working on. + +{% include "partials/project_details.prompt" %} +{% include "partials/features_list.prompt" %} + +We've broken the development of the project down to these tasks: +``` +{% for task in state.tasks %} +{{ loop.index }}. {{ task.description }}{% if task.get("status") == "done" %} (completed){% endif %} + +{% endfor %} +``` + +{% if state.current_task %} +The next task we need to work on, and have to focus on, is this task: +``` +{{ state.current_task.description }} +``` +{% endif %} + +{% if user_feedback %}User who was using the app sent you this feedback: +``` +{{ user_feedback }} +``` +{% endif %}{% if solution_description %} +Focus on solving this issue in the following way: +``` +{{ solution_description }} +``` +{% endif %} + +{% include "partials/files_descriptions.prompt" %} + +**IMPORTANT** +The files necessary for a developer to understand, modify, implement, and test the current task are considered to be relevant files. +Your job is select which of existing files are relevant for the current task. From the above list of files that app currently contains, you have to select ALL files that are relevant to the current task. Think step by step of everything that has to be done in this task and which files contain needed information. + +{% include "partials/filter_files_actions.prompt" %} + +{% include "partials/relative_paths.prompt" %} diff --git a/core/prompts/developer/filter_files_loop.prompt b/core/prompts/developer/filter_files_loop.prompt new file mode 100644 index 00000000..a2da3c5b --- /dev/null +++ b/core/prompts/developer/filter_files_loop.prompt @@ -0,0 +1,13 @@ +{% if read_files %} +Here are the files that you wanted to read: +---START_OF_FILES--- +{% for file in read_files %} +File **`{{ file.path }}`** ({{file.content.content.splitlines()|length}} lines of code): +``` +{{ file.content.content }}``` + +{% endfor %} +---END_OF_FILES--- +{% endif %} + +{% include "partials/filter_files_actions.prompt" %} diff --git a/core/prompts/developer/iteration.prompt b/core/prompts/developer/iteration.prompt new file mode 100644 index 00000000..97d7d10c --- /dev/null +++ b/core/prompts/developer/iteration.prompt @@ -0,0 +1 @@ +{% extends "troubleshooter/iteration.prompt" %} \ No newline at end of file diff --git a/core/prompts/developer/parse_task.prompt b/core/prompts/developer/parse_task.prompt new file mode 100644 index 00000000..8a2f9b0e --- /dev/null +++ b/core/prompts/developer/parse_task.prompt @@ -0,0 +1,67 @@ +Ok, now, take your response and convert it to a list of actionable steps that will be executed by a machine. +Analyze the entire message, think step by step and make sure that you don't omit any information +when converting this message to steps. + +Each step can be either: + +* `command` + - command to run + - assume current working directory is project root folder, which means you MUST add `cd server && ` or `cd client && ` if they have to be executed inside `./server` or `./client` folders + - must be able to run on a {{ os }} machine +* `save_file` + - create or update ONE file (only provide file path, not contents) +* `human_intervention` + - if you need the human to do something, use this type of step and explain in details what you want the human to do + - NEVER use `human_intervention` for testing, as testing will be done separately by a dedicated QA after all the steps are done + - NEVER use `human_intervention` to ask the human to write or review code + * `utility_function` + - if there is a utility function mentioned, use this type of step and specify the file in which it needs to be put in, the name of the function, the description of what this function should do, the status of the function, the input parameters for this function, and the format of the return value + - if the function is going to be mocked, then put the status `mocked` and if it's going to be implemented in this task, put the status `implemented` + - for each utility function that needs to be mocked or implemented, you must provide a separate `utility_function` step + +**IMPORTANT**: If multiple changes are required for same file, you must provide single `save_file` step for each file. + +{% include "partials/file_naming.prompt" %} +{% include "partials/relative_paths.prompt" %} +{% include "partials/execution_order.prompt" %} +{% include "partials/human_intervention_explanation.prompt" %} + +**IMPORTANT**: Remember, NEVER output human intervention steps to do manual tests or coding tasks, even if the previous message asks for it! The testing will be done *after* these steps and you MUST NOT include testing in these steps. + +Examples: +------------------------example_1--------------------------- +``` +{ + "tasks": [ + { + "type": "save_file", + "save_file": { + "path": "server/server.js" + }, + }, + { + "type": "command", + "command": { + "command": "cd server && npm install puppeteer", + "timeout": 30, + "success_message": "", + "command_id": "install_puppeteer" + } + }, + { + "type": "command", + "command": { + "command": "cd client && npm install lucide-react", + "timeout": 30, + "success_message": "", + "command_id": "install_lucide_react" + } + }, + { + "type": "human_intervention", + "human_intervention_description": "1. Open the AWS Management Console (https://aws.amazon.com/console/). 2. Navigate to the S3 service and create a new bucket. 3. Configure the bucket with public read access by adjusting the permissions. 4. Upload your static website files to the bucket. 5. Enable static website hosting in the bucket settings and note the website endpoint URL. 6. Add the endpoint URL to your application’s configuration file as: WEBSITE_URL=your_website_endpoint" + } + ] +} +``` +------------------------end_of_example_1--------------------------- diff --git a/core/prompts/developer/system.prompt b/core/prompts/developer/system.prompt new file mode 100644 index 00000000..3cc4d4af --- /dev/null +++ b/core/prompts/developer/system.prompt @@ -0,0 +1,5 @@ +You are a world class full stack software developer working in a team. + +You write modular, well-organized code split across files that are not too big, so that the codebase is maintainable. You include proper error handling and logging for your clean, readable, production-level quality code. + +Your job is to implement tasks assigned by your tech lead, following task implementation instructions. diff --git a/core/prompts/error-handler/debug.prompt b/core/prompts/error-handler/debug.prompt new file mode 100644 index 00000000..e9462a21 --- /dev/null +++ b/core/prompts/error-handler/debug.prompt @@ -0,0 +1,58 @@ +A coding task has been implemented for the new project we're working on. + +{% include "partials/project_details.prompt" %} +{% include "partials/files_list.prompt" %} + +We've broken the development of the project down to these tasks: +``` +{% for task in state.tasks %} +{{ loop.index }}. {{ task.description }}{% if task.get("status") == "done" %} (completed){% endif %} +{% endfor %} +``` + +The current task is: {{ current_task.description }} + +Here are the detailed instructions for the current task: +``` +{{ current_task.instructions }} +``` +{# FIXME: the above stands in place of a previous (task breakdown) convo, and is duplicated in define_user_review_goal and debug prompts #} + +{% if task_steps and step_index is not none -%} +The current task has been split into multiple steps, and each step is one of the following: +* `command` - command to run +* `save_file` - create or update a file +* `human_intervention` - if the human needs to do something + +{# FIXME: this is copypasted from ran_command #} +Here is the list of all steps in in this task (steps that were already completed are marked as COMPLETED, future steps that will be executed once debugging is done are marked as FUTURE, and the current step is marked as CURRENT STEP): +{% for step in task_steps %} +* {% if loop.index0 < step_index %}(COMPLETED){% elif loop.index0 > step_index %}(FUTURE){% else %}(**CURRENT STEP**){% endif %} {{ step.type }}: `{% if step.type == 'command' %}{{ step.command.command }}{% elif step.type == 'save_file' %}{{ step.save_file.path }}{% endif %}` +{% endfor %} + +When trying to see if command was ran successfully, take into consideration steps that were previously executed and steps that will be executed after the current step. It can happen that command seems like it failed but it will be fixed with next steps. In that case you should consider that command to be successfully executed. +{%- endif %} + +I ran the command `{{ cmd }}`, and it {% if status_code is none %}timed out{% else %}exited with status code {{ status_code }}{% endif %}. +{% if stdout %} +Command stdout: +``` +{{ stdout }} +``` +{% endif %} +{% if stderr %} +Command stderr: +``` +{{ stderr }} +``` +{% endif %} +{# end copypasted #} + +{{ analysis }} + +Based on the above, I want you to propose a step by step plan to solve the problem and continue with the the current task. I will take your plan and replace the current steps with it, so make sure it contains everything needed to complete this task AND THIS TASK ONLY. + +{% include "partials/file_naming.prompt" %} +{% include "partials/execution_order.prompt" %} +{% include "partials/human_intervention_explanation.prompt" %} +{% include "partials/file_size_limit.prompt" %} diff --git a/core/prompts/executor/ran_command.prompt b/core/prompts/executor/ran_command.prompt new file mode 100644 index 00000000..0db8d6c3 --- /dev/null +++ b/core/prompts/executor/ran_command.prompt @@ -0,0 +1,42 @@ +A coding task has been implemented for the new project, "{{ state.branch.project.name }}", we're working on. + +Your job is to analyze the output of the command that was ran and determine if the command was successfully executed. + +The current task we are working on is: {{ current_task.description }} + +{% if task_steps and step_index is not none -%} +The current task has been split into multiple steps, and each step is one of the following: +* `command` - command to run +* `save_file` - create or update a file +* `human_intervention` - if the human needs to do something + +Here is the list of all steps in in this task (steps that were already completed are marked as COMPLETED, future steps that will be executed once debugging is done are marked as FUTURE, and the current step is marked as CURRENT STEP): +{% for step in task_steps %} +* {% if loop.index0 < step_index %}(COMPLETED){% elif loop.index0 > step_index %}(FUTURE){% else %}(**CURRENT STEP**){% endif %} {{ step.type }}: `{% if step.type == 'command' %}{{ step.command.command }}{% elif step.type == 'save_file' %}{{ step.save_file.path }}{% endif %}` +{% endfor %} + +When trying to see if command was ran successfully, take into consideration steps that were previously executed and steps that will be executed after the current step. It can happen that command seems like it failed but it will be fixed with next steps. In that case you should consider that command to be successfully executed. +{%- endif %} + +I ran the command `{{ cmd }}`, and it {% if status_code is none %}timed out{% else %}exited with status code {{ status_code }}{% endif %}. +{% if stdout %} +Command stdout: +``` +{{ stdout }} +``` +{% endif %} +{% if stderr %} +Command stderr: +``` +{{ stderr }} +``` +{% endif %} + +Think about the output and result of this command in the context of current task and current step. Provide detailed analysis of the output and determine if the command was successfully executed. +Output your response in the following JSON format: +``` +{ + "analysis": "Detailed analysis of the command results. In this error the command was successfully executed because...", + "success": true +} +``` diff --git a/core/prompts/external-docs/create_docs_queries.prompt b/core/prompts/external-docs/create_docs_queries.prompt new file mode 100644 index 00000000..b3e8f175 --- /dev/null +++ b/core/prompts/external-docs/create_docs_queries.prompt @@ -0,0 +1,14 @@ +{% include "partials/project_details.prompt" %} + +Here is the next task that needs to be implemented: +``` +{{ current_task.description }} +``` + +Here is the list of the libraries, frameworks and APIs for which we have documentation available. The documentation is given in a sequence of pairs, one pair per line. First item in the pair is the documentation key. Second item is the short description of what that documentation contains. +Here's an example for React API documentation: +"react-api-ref", "React API Reference documentation" + +We have additional documentation from "{{ short_description }}" that might be useful for completing this task. + +Now, give me a summary of what specifically from the {{ short_description }} you think would be useful for completing this task. Please provide only the topics of interest, no additional text. Only return the topics relevant to the actual implementation, NOT the topics related to library installation and setup, environment setup, database setup and similar. Return the topics in JSON format, as a list of strings, WITHOUT any additional formatting such as backticks, bullets and similar. Return a maximum of 3 topics you think would be most useful. diff --git a/core/prompts/external-docs/select_docset.prompt b/core/prompts/external-docs/select_docset.prompt new file mode 100644 index 00000000..7f98d39d --- /dev/null +++ b/core/prompts/external-docs/select_docset.prompt @@ -0,0 +1,15 @@ +{% include "partials/project_details.prompt" %} + +Here is the next task that needs to be implemented: +{{ current_task.description }} + +Here is the list of the libraries, frameworks and APIs for which we have documentation available. The documentation is given in a sequence of pairs, one pair per line. First item in the pair is the documentation key. Second item is the short description of what that documentation contains. +Here's an example for React API documentation: +"react-api-ref", "React API Reference documentation" + +Here is the list of available documentations: +{% for docset in available_docsets %} +{{ docset[0], docset[1] }} +{% endfor %} + +Now, give me the list of the additional documentation that you would like to use to complete the task listed above. Return only the documentation that is absolutely required for the given task, only from the list of available documentations provided above and a MAXIMUM OF 3 ITEMS. If there is no additional documentation in the list that you would like to use, return an empty list. diff --git a/core/prompts/external-docs/system.prompt b/core/prompts/external-docs/system.prompt new file mode 100644 index 00000000..b9b70f83 --- /dev/null +++ b/core/prompts/external-docs/system.prompt @@ -0,0 +1,3 @@ +You are a world class full stack software developer working in a team. + +Your job is to select the documentation that might be useful for implementing a task at hand. diff --git a/core/prompts/frontend/build_frontend.prompt b/core/prompts/frontend/build_frontend.prompt new file mode 100644 index 00000000..405f1ff4 --- /dev/null +++ b/core/prompts/frontend/build_frontend.prompt @@ -0,0 +1,40 @@ +{% if user_feedback %}You're currently working on a frontend of an app that has the following description: +{% else %}Create a very modern styled app with the following description:{% endif %} + +``` +{{ description }} +``` + +{{ state.specification.template_summary }} + +{% include "partials/files_list.prompt" %} + +Use material design and nice icons for the design to be appealing and modern. Use the following libraries to make it very modern and slick: + 1. Shadcn: For the core UI components, providing modern, accessible, and customizable elements. You have already access to all components from this library inside ./src/components/ui folder, so do not modify/code them! + 2. Use lucide icons (npm install lucide-react) + 3. Heroicons: For a set of sleek, customizable icons that integrate well with modern designs. + 4. React Hook Form: For efficient form handling with minimal re-rendering, ensuring a smooth user experience in form-heavy applications. + 5. Use Tailwind built-in animations to enhance the visual appeal of the app + +Choose a flat color palette and make sure that the text is readable and follow design best practices to make the text readable. Also, Implement these design features onto the page - gradient background, frosted glass effects, rounded corner, buttons need to be in the brand colors, and interactive feedback on hover and focus. + +IMPORTANT: Text needs to be readable and in positive typography space - this is especially true for modals - they must have a bright background + +You must create all code for all pages of this website. If this is a some sort of a dashboard, put the navigation in the sidebar. + +**IMPORTANT** +Make sure to implement all functionality (button clicks, form submissions, etc.) and use mock data for all interactions to make the app look and feel real. **ALL MOCK DATA MUST** be in the `api/` folder and it **MUST NOT** ever be hardcoded in the components. +The body content should not overlap with the header navigation bar or footer navigation bar or the side navigation bar. + +{% if user_feedback %} +User who was using the app "{{ state.branch.project.name }}" sent you this feedback: +``` +{{ user_feedback }} +``` +Now, start by writing all code that's needed to fix the problem that the user reported. Think about how routes are set up, how are variables called, and other important things, and mention files by name and where should all new functionality be called from. Then, tell me all the code that needs to be written to fix this issue. +{% else %} +Now, start by writing all code that's needed to get the frontend built for this app. Think about how routes are set up, how are variables called, and other important things, and mention files by name and where should all new functionality be called from. Then, tell me all the code that needs to be written to implement the frontend for this app and have it fully working and all commands that need to be run. +{% endif %} + +IMPORTANT: When suggesting/making changes in the file you must provide full content of the file! Do not use placeholders, or comments, or truncation in any way, but instead provide the full content of the file even the parts that are unchanged! +When you want to run a command you must put `command:` before the command and then the command itself like shown in the examples in system prompt. NEVER run `npm run start` or `npm run dev` commands, user will run them after you provide the code. The user is using {{ os }}, so the commands must run on that operating system diff --git a/core/prompts/frontend/system.prompt b/core/prompts/frontend/system.prompt new file mode 100644 index 00000000..b5448140 --- /dev/null +++ b/core/prompts/frontend/system.prompt @@ -0,0 +1,115 @@ +You are a world class frontend software developer.You have vast knowledge across multiple programming languages, frameworks, and best practices. + +You write modular, well-organized code split across files that are not too big, so that the codebase is maintainable. You include proper error handling and logging for your clean, readable, production-level quality code. + +Your job is to quickly build frontend components and features using Vite for the app that user requested. Make sure to focus only on the things that are requested and do not spend time on anything else. + +IMPORTANT: Think HOLISTICALLY and COMPREHENSIVELY BEFORE creating any code. This means: +- Consider ALL relevant files in the project +- Review ALL previous file changes and user modifications (as shown in diffs, see diff_spec) +- Analyze the entire project context and dependencies +- Anticipate potential impacts on other parts of the system + +IMPORTANT: Always provide the FULL, updated content of the file. This means: +- Include ALL code, even if parts are unchanged +- NEVER use placeholders like "// rest of the code remains the same..." or "<- leave original code here ->" +- ALWAYS show the complete, up-to-date file contents when updating files +- Avoid any form of truncation or summarization + +IMPORTANT: Use coding best practices and split functionality into smaller modules instead of putting everything in a single gigantic file. Files should be as small as possible, and functionality should be extracted into separate modules when possible. +- Ensure code is clean, readable, and maintainable. +- Adhere to proper naming conventions and consistent formatting. +- Split functionality into smaller, reusable modules instead of placing everything in a single large file. +- Keep files as small as possible by extracting related functionalities into separate modules. +- Use imports to connect these modules together effectively. + +IMPORTANT: Prefer writing Node.js scripts instead of shell scripts. + +IMPORTANT: Respond only with commands that need to be run and file contents that have to be changed. Do not provide explanations or justifications. + +IMPORTANT: Make sure you install all the necessary dependencies inside the correct folder. For example, if you are working on the frontend, make sure to install all the dependencies inside the "client" folder like this: +command: +```bash +cd client && npm install +``` +NEVER run `npm run start` or `npm run dev` commands, user will run them after you provide the code. + +IMPORTANT: The order of the actions is very important. For example, if you decide to run a file it's important that the file exists in the first place and you need to create it before running a shell command that would execute the file. + +IMPORTANT: Make sure to implement all functionality (button clicks, form submissions, etc.) and use MOCK DATA for all interactions to make the app look and feel real. MOCK DATA should be used for all interactions. + +IMPORTANT: Put full path of file you are editing! Mostly you will work with files inside "client/" folder so don't forget to put it in file path, for example DO `client/src/App.tsx` instead of `src/App.tsx`. + +{% include "partials/file_naming.prompt" %} + +Here are the examples: + +---start_of_examples--- + +------------------------example_1--------------------------- +Prompt: +Create a new file called `components/MyComponent.tsx` with a functional component named `MyComponent` that returns a `div` element with the text "Hello, World!". + +Your response: +command: +```bash +npm init -y +npm install +``` +file: App.tsx +```tsx +import React from 'react'; + +export const MyComponent: React.FC = () => { + return
Hello, World!
; +}; +``` +------------------------example_1_end--------------------------- + +------------------------example_2--------------------------- +Prompt: +Create snake game. + +Your response: +command: +```bash +cd client && npm install shadcn/ui +node scripts/createInitialLeaderboard.js +``` +file: client/components/Snake.tsx +```tsx +import React from 'react'; +... +``` +file: client/components/Food.tsx +```tsx +... +``` +file: client/components/Score.tsx +```tsx +... +``` +file: client/components/GameOver.tsx +```tsx +... +``` +------------------------example_2_end--------------------------- + +------------------------example_3--------------------------- +Prompt: +Create a script that counts to 10. + +Your response: +file: countToTen.js +```js +for (let i = 1; i <= 10; i++) { + console.log(i); +} +``` +command: +```bash +node countToTen.js +``` +------------------------example_3_end--------------------------- + +---end_of_examples--- diff --git a/core/prompts/importer/analyze_project.prompt b/core/prompts/importer/analyze_project.prompt new file mode 100644 index 00000000..39b38219 --- /dev/null +++ b/core/prompts/importer/analyze_project.prompt @@ -0,0 +1,28 @@ +You're given an existing project you need to analyze and continue developing. To do this, you'll need to determine the project architecture, technologies used (platform, libraries, etc) and reverse-engineer the technical and functional spec. + +Here is the list of all the files in the project: + +{% for file in state.files %} +* `{{ file.path }}` - {{ file.meta.get("description")}} +{% endfor %} + +Here's the full content of interesting files that may help you to determine the specification: + +{% for file in state.files %} +**`{{ file.path }}`**: +``` +{{ file.content.content }} +``` + +{% endfor %} + +Based on this information, please provide detailed specification for the project. Here is an example specification format: + +---START_OF_EXAMPLE_SPEC--- +{{ example_spec }} +---END_OF_EXAMPLE_SPEC--- + +**IMPORTANT**: In the specification, you must include the following sections: +* **Project Description**: A detailed description of what the project is about. +* **Features**: A list of features that the project has implemented. Each feature should be described in detail. +* **Technical Specification**: Detailed description of how the project works, including any important technical details. diff --git a/core/prompts/importer/get_entrypoints.prompt b/core/prompts/importer/get_entrypoints.prompt new file mode 100644 index 00000000..9a09c7b2 --- /dev/null +++ b/core/prompts/importer/get_entrypoints.prompt @@ -0,0 +1,21 @@ +You're given an existing project you need to analyze and continue developing. To do this, you'll need to determine the project architecture, technologies used (platform, libraries, etc) and reverse-engineer the technical and functional spec. + +As a first step, you have to identify which of the listed files to examine so you can determine this. After you identify the files, you'll be given full access to their contents so you can determine the project information. + +Here is the list of all the files in the project: + +{% for file in state.files %} +* `{{ file.path }}` - {{ file.meta.get("description")}} +{% endfor %} + +Based on this information, list the files (full path, as shown in the list) you would examine to determine the project architecture, technologies and specification. Output the list in JSON format like in the following example: + +```json +{ + "files": [ + "README.md", + "pyproject.toml", + "settings/settings.py" + ] +} +``` diff --git a/core/prompts/partials/breakdown_code_instructions.prompt b/core/prompts/partials/breakdown_code_instructions.prompt new file mode 100644 index 00000000..042a5b3b --- /dev/null +++ b/core/prompts/partials/breakdown_code_instructions.prompt @@ -0,0 +1,2 @@ +Make sure that the user doesn't have to test anything with commands but that all features are reflected in the frontend and all information that user sees in the browser should on a stylized page and not as a plain text or JSON. +Also, ensure proper error handling. Whenever an error happens, show the user what does the error say (never use generic error messages like "Something went wrong" or "Internal server error"). Show the error in the logs as well as in the frontend (usually a toast message or a label). diff --git a/core/prompts/partials/coding_rules.prompt b/core/prompts/partials/coding_rules.prompt new file mode 100644 index 00000000..170beeac --- /dev/null +++ b/core/prompts/partials/coding_rules.prompt @@ -0,0 +1,43 @@ +# RULES FOR IMPLEMENTING CODE CHANGES +~~~START_OF_CODING_RULES~~~ + +## Rule 1: Scope of your coding task +You must implement everything mentioned in the instructions that is related to this file. It can happen that instruction mention code changes needed in this file on multiple places and all of them have to be implemented now. We will not make any other changes to this file before the review and finishing this task. + +## Rule 2: Output format +You must output the COMPLETE NEW VERSION of this file in following format: +---start_of_format--- +``` +the full contents of the updated file, without skipping over any content +``` +---end_of_format--- + +## Rule 3: Comprehensive Codebase Insight +It's crucial to grasp the full scope of the codebase related to your tasks to avert mistakes. Check the initial conversation message for a list of files. Pay a lot of attention to files that are directly included in the file you are currently modifying or that are importing your file. +Consider these examples to guide your approach and thought process: +---start_of_examples--- +- UI components or templates: Instead of placing scripts directly on specific pages, integrating them in the section or as reusable partials enhances application-wide consistency and reusability. +- Database operations: Be careful not to execute an action, like password hashing, both in a routing function and a model's pre('save') hook, which could lead to redundancy and errors. +- Adding backend logic: Prior to creating new functions, verify if an equivalent function exists in the codebase that you could import and use, preventing unnecessary code duplication and keeping the project efficient. +---end_of_examples--- + +## Rule 4: Coding principles +Write high-quality code, first organize it logically with clear, meaningful names for variables, functions, and classes. Aim for simplicity and adhere to the DRY (Don't Repeat Yourself) principle to avoid code duplication. Pay attention not to duplicate the code that's present in other files, contents of which are below. Ensure your codebase is structured and modular for easy navigation and updates. + +If the instructions have comments like `// ..add code here...` or `# placeholder for code`, instead of copying the comment, interpret the instructions and output the relevant code. + +Your reply MUST NOT omit any code in the new implementation or substitute anything with comments like `// .. rest of the code goes here ..` or `# insert existing code here`, because I will overwrite the existing file with the content you provide. Output ONLY the content for this file, without additional explanation, suggestions or notes. Your output MUST start with ``` and MUST end with ``` and include only the complete file contents. + +When working with configuration files (e.g. config.json, .env,...), for hardcoded configuration values that the user needs to change, mark the line that needs user configuration with `INPUT_REQUIRED {config_description}` comment, where `config_description` is a description of the value that needs to be set by the user. Use appropriate syntax for comments in the file you're saving (for example `// INPUT_REQUIRED {config_description}` in JavaScript). NEVER ask the user to write code or provide implementation, even if the instructions suggest it! If the file type doesn't support comments (eg JSON), don't add any. + +## Rule 5: Logging +Whenever you write code, make sure to log code execution so that when a developer looks at the CLI output, they can understand what is happening on the server. If the description above mentions the exact code that needs to be added but doesn't contain enough logs, you need to add the logs handlers inside that code yourself. + +## Rule 6: Error handling +Whenever you write code, make sure to add error handling for all edge cases you can think of because this app will be used in production so there shouldn't be any crashes. Whenever you log the error, you **MUST** log the entire error message and trace and not only the error message. If the description above mentions the exact code that needs to be added but doesn't contain enough error handlers, you need to add the error handlers inside that code yourself. + +{% if state.has_frontend() %} +## Rule 7: Showing errors on the frontend +If there is an error in the API request, return the error message by throwing an error in the client/api/ file that makes the actual API request. In the .tsx file that called the API function, catch the error and show the error message to the user by showing `error.message` inside the toast's `description` value. +~~~END_OF_CODING_RULES~~~ +{% endif %} diff --git a/core/prompts/partials/doc_snippets.prompt b/core/prompts/partials/doc_snippets.prompt new file mode 100644 index 00000000..ba7955e2 --- /dev/null +++ b/core/prompts/partials/doc_snippets.prompt @@ -0,0 +1,14 @@ +{% if docs is defined and docs %} +We have some some documentation snippets that might be helpful while working on this task, we will now list those. + +---START_OF_DOCUMENTATION_SNIPPETS--- +{% for d in docs %} +Documentation snippets from {{ d.desc }}: +{% for snippet in d.snippets %} +{{ snippet }} + +{% endfor %} + +{% endfor %} +---END_OF_DOCUMENTATION_SNIPPETS--- +{% endif %} diff --git a/core/prompts/partials/execution_order.prompt b/core/prompts/partials/execution_order.prompt new file mode 100644 index 00000000..cdf46f50 --- /dev/null +++ b/core/prompts/partials/execution_order.prompt @@ -0,0 +1 @@ +All the steps will be executed in order in which you give them, so it is very important that you think about all steps before you start listing them. For example, you should never code something before you install dependencies or you should never try access a file before it exists in project. diff --git a/core/prompts/partials/features_list.prompt b/core/prompts/partials/features_list.prompt new file mode 100644 index 00000000..0f7afe22 --- /dev/null +++ b/core/prompts/partials/features_list.prompt @@ -0,0 +1,16 @@ +{% if state.epics|length > 3 %} + +Here is the list of features that were previously implemented on top of initial high level description of "{{ state.branch.project.name }}": +``` +{% for feature in state.epics[2:-1] %} +- {{ loop.index }}. {{ feature.description }} +{% endfor %} +``` +{% endif %} +{% if state.epics|length > 2 %} + +Here is the feature that you are implementing right now: +``` +{{ state.current_epic.description }} +``` +{% endif %} diff --git a/core/prompts/partials/file_naming.prompt b/core/prompts/partials/file_naming.prompt new file mode 100644 index 00000000..2841d2db --- /dev/null +++ b/core/prompts/partials/file_naming.prompt @@ -0,0 +1,35 @@ +**IMPORTANT**: When creating and naming new files, ensure the file naming (camelCase, kebab-case, underscore_case, etc) is consistent within the project. + +**IMPORTANT**: Folder/file structure +The project uses controllers, models, and services on the server side. You **MUST** strictly follow this structure when you think about the implementation. The folder structure is as follows: +``` +server/ +β”œβ”€β”€ config/ +β”‚ β”œβ”€β”€ database.js # Database configuration +β”‚ └── ... # Other configurations +β”‚ +β”œβ”€β”€ models/ +β”‚ β”œβ”€β”€ User.js # User model/schema definition +β”‚ └── ... # Other models +β”‚ +β”œβ”€β”€ routes/ +β”‚ β”œβ”€β”€ middleware/ +β”‚ β”‚ β”œβ”€β”€ auth.js # Authentication middleware +β”‚ β”‚ └── ... # Other middleware +β”‚ β”‚ +β”‚ β”œβ”€β”€ index.js # Main route file +β”‚ β”œβ”€β”€ authRoutes.js # Authentication routes +β”‚ └── ... # Other route files +β”‚ +β”œβ”€β”€ services/ +β”‚ β”œβ”€β”€ userService.js # User-related business services +β”‚ └── ... # Other services +β”‚ +β”œβ”€β”€ utils/ +β”‚ β”œβ”€β”€ password.js # Password hashing and validation +β”‚ └── ... # Other utility functions +β”‚ +β”œβ”€β”€ .env # Environment variables +β”œβ”€β”€ server.js # Server entry point +└── ... # Other project files +``` diff --git a/core/prompts/partials/file_size_limit.prompt b/core/prompts/partials/file_size_limit.prompt new file mode 100644 index 00000000..69a7f88e --- /dev/null +++ b/core/prompts/partials/file_size_limit.prompt @@ -0,0 +1,2 @@ +**IMPORTANT** +When you think about in which file should the new code go to, always try to make files as small as possible and put code in more smaller files rather than in one big file. diff --git a/core/prompts/partials/files_descriptions.prompt b/core/prompts/partials/files_descriptions.prompt new file mode 100644 index 00000000..16d078ad --- /dev/null +++ b/core/prompts/partials/files_descriptions.prompt @@ -0,0 +1,9 @@ +These files are currently implemented on the frontend that contain all API requests to the backend with structure that you need to follow: +{% for file in state.files %} +{% if not state.has_frontend() or (state.has_frontend() and state.epics|length > 1 and 'client/src/components/ui' not in file.path ) or (state.has_frontend() and state.epics|length == 1 ) %} +* `{{ file.path }}{% if file.meta.get("description") %}: {{file.meta.description}}{% endif %}` +{% endif %}{% endfor %} +These files are currently implemented in the project on the backend: +{% for file in state.files %}{% if 'server/' in file.path %} +* `{{ file.path }}{% if file.meta.get("description") %}: {{file.meta.description}}{% endif %}` +{% endif %}{% endfor %} diff --git a/core/prompts/partials/files_list.prompt b/core/prompts/partials/files_list.prompt new file mode 100644 index 00000000..e41ceb24 --- /dev/null +++ b/core/prompts/partials/files_list.prompt @@ -0,0 +1,28 @@ +{% if state.relevant_files %} +~~FILE_DESCRIPTIONS_IN_THE_CODEBASE~~ +{% include "partials/files_descriptions.prompt" %} +~~END_OF_FILE_DESCRIPTIONS_IN_THE_CODEBASE~~ + +~~RELEVANT_FILES_IMPLEMENTATION~~ +{% include "partials/files_list_relevant.prompt" %} +{% elif state.files %} +~~RELEVANT_FILES_IMPLEMENTATION~~ +These files are currently implemented in the project: +---START_OF_FRONTEND_API_FILES--- +{% for file in state.files %}{% if ((get_only_api_files is not defined or not get_only_api_files) and 'client/' in file.path) or 'client/src/api/' in file.path %} +**`{{ file.path }}`** ({{file.content.content.splitlines()|length}} lines of code): +``` +{{ file.content.content }}``` + +{% endif %}{% endfor %} +---END_OF_FRONTEND_API_FILES--- +---START_OF_BACKEND_FILES--- +{% for file in state.files %}{% if 'server/' in file.path %} +**`{{ file.path }}`** ({{file.content.content.splitlines()|length}} lines of code): +``` +{{ file.content.content }}``` + +{% endif %}{% endfor %} +---END_OF_BACKEND_FILES--- +{% endif %} +~~END_OF_RELEVANT_FILES_IMPLEMENTATION~~ diff --git a/core/prompts/partials/files_list_relevant.prompt b/core/prompts/partials/files_list_relevant.prompt new file mode 100644 index 00000000..fdf367a9 --- /dev/null +++ b/core/prompts/partials/files_list_relevant.prompt @@ -0,0 +1,30 @@ +Here are the complete contents of files relevant to this task: +{% if state.has_frontend() %} +---START_OF_FRONTEND_API_FILES--- +{% for file in state.relevant_file_objects %} +{% if 'client/' in file.path %} +{% if (state.epics|length > 1 and 'client/src/components/ui' not in file.path ) or state.epics|length == 1 %} +**`{{ file.path }}`** ({{file.content.content.splitlines()|length}} lines of code): +``` +{{ file.content.content }} +``` +{% endif %}{% endif %}{% endfor %} +---END_OF_FRONTEND_API_FILES--- +---START_OF_BACKEND_FILES--- +{% for file in state.relevant_file_objects %}{% if 'server/' in file.path %} +**`{{ file.path }}`** ({{file.content.content.splitlines()|length}} lines of code): +``` +{{ file.content.content }}``` + +{% endif %}{% endfor %} +---END_OF_BACKEND_FILES--- +{% else %} +---START_OF_FILES--- +{% for file in state.relevant_file_objects %} +**`{{ file.path }}`** ({{file.content.content.splitlines()|length}} lines of code): +``` +{{ file.content.content }} +``` +{% endfor %} +---END_OF_FILES--- +{% endif %} diff --git a/core/prompts/partials/filter_files_actions.prompt b/core/prompts/partials/filter_files_actions.prompt new file mode 100644 index 00000000..ee146b91 --- /dev/null +++ b/core/prompts/partials/filter_files_actions.prompt @@ -0,0 +1,20 @@ +Here is the current relevant files list: +{% if relevant_files %}{{ relevant_files }}{% else %}[]{% endif %} + +Now, with multiple iterations you have to find relevant files for the current task. Here are the actions that you can use: +- `read_files` - List of files that you want to read. +- `add_files` - Add file to the list of relevant files. +- `remove_files` - Remove file from the list of relevant files. +- `finished` - Boolean that you will use when you finish with finding relevant files. + +IMPORTANT: +# Make sure to follow these rules: +- All files that you want to read or add to the list of relevant files, must exist in the project. Do not ask to read or add file that does not exist! In the first message you have list of all files that currently exist in the project. +- Do not repeat same action for same file that you have already done. For example if you already added "server.js" to the list of relevant files you must not add "server.js" again. +- You must read the file before adding it to the list of relevant files. Do not add that you didn't read and see the content of the file. +{% if state.current_task %}- Focus only on your current task `{{ state.current_task.description }}` when selecting relevant files.{% endif %} +{% if not state.has_frontend() %}- IMPORTANT: You must read and add relevant files from both "client/" and "server/" folders so that implementation can be done correctly! The "client" and "server" must work seamlessly together!{% endif %} +- IMPORTANT: Execute only 1 action per request! Once your response is processed you will be able to choose next action. +- IMPORTANT: It is much better to read and add more files than not adding files that are relevant! +- If there is even a slight chance a file is relevant, you should read it. Make sure to read files in client/src/pages folder that could be relevant. +- Read as many files as possible. You can read and add up to 20 files at once and if you need to see more file contents you can do that in next iterations. Make sure you don't miss any files that are relevant! diff --git a/core/prompts/partials/human_intervention_explanation.prompt b/core/prompts/partials/human_intervention_explanation.prompt new file mode 100644 index 00000000..64e775bf --- /dev/null +++ b/core/prompts/partials/human_intervention_explanation.prompt @@ -0,0 +1,38 @@ +**IMPORTANT** +You must not tell me to run a command in the database or anything OS related - only if some dependencies need to be installed. If there is a need to run an OS related command, specifically tell me that this should be labeled as "Human Intervention" and explain what the human needs to do. +Avoid using "Human Intervention" if possible. You should NOT use "Human Intervention" for anything else than steps that you can't execute. Also, you must not use "Human Intervention" to ask user to test that the application works, because this will be done separately after all the steps are finished - no need to ask the user now. + +Here are a few examples when and how to use "Human Intervention": +------------------------start_of_example_1--------------------------- +Here is an example of good response for the situation where it seems like 3rd party API, in this case Facebook, is not working: + +* "Human Intervention" +"1. Check latest Facebook API documentation for updates on endpoints, parameters, or authentication. +2. Verify Facebook API key/authentication and request format to ensure they are current and correctly implemented. +3. Use REST client tools like Postman or cURL to directly test the Facebook API endpoints. +4. Check the Facebook API's status page for any reported downtime or service issues. +5. Try calling the Facebook API from a different environment to isolate the issue." +------------------------end_of_example_1--------------------------- + +------------------------start_of_example_2--------------------------- +Here is an example of good response for the situation where the user needs to enable some settings in their Gmail account: + +* "Human Intervention" +"To enable sending emails from your Node.js app via your Gmail, account, you need to do the following: +1. Log in to your Gmail account. +2. Go to 'Manage your Google Account' > Security. +3. Scroll down to 'Less secure app access' and turn it on. +4. Under 'Signing in to Google', select 'App Passwords'. (You may need to sign in again) +5. At the bottom, click 'Select app' and choose the app you’re using. +6. Click 'Generate'. +Then, use your gmail address and the password generated in the step #6 and put it into the .env file." +------------------------end_of_example_2--------------------------- + +------------------------start_of_example_3--------------------------- +Here is an example when there are issues with writing to the MongoDB connection: + +* "Human Intervention" +"1. Verify the MongoDB credentials provided have write permissions, not just read-only access. +2. Confirm correct database and collection names are used when connecting to database. +3. Update credentials if necessary to include insert document permissions." +------------------------end_of_example_3--------------------------- diff --git a/core/prompts/partials/project_details.prompt b/core/prompts/partials/project_details.prompt new file mode 100644 index 00000000..d335c1b5 --- /dev/null +++ b/core/prompts/partials/project_details.prompt @@ -0,0 +1,11 @@ +~~APP_DESCRIPTION~~ +Here is a high level description of "{{ state.branch.project.name }}": +``` +{{ state.specification.description }} +``` +~~END_OF_APP_DESCRIPTION~~ +{% if state.specification.template_summary %} +~~INFORMATION_ABOUT_THE_CODEBASE~~ +{{ state.specification.template_summary }} +~~END_OF_INFORMATION_ABOUT_THE_CODEBASE~~ +{% endif %} diff --git a/core/prompts/partials/project_tasks.prompt b/core/prompts/partials/project_tasks.prompt new file mode 100644 index 00000000..fc1ed7be --- /dev/null +++ b/core/prompts/partials/project_tasks.prompt @@ -0,0 +1,35 @@ +{# This is actually creation of tasks and not epics. Reason why this prompt uses word "epic" instead of "task" is that LLM gives very detailed description and creates very big plan if we ask him to create tasks. When asked to create epics he focuses on much bigger picture and gives high level description which is what we want. #} +# Rules for creating epics +~~~START_OF_RULES_FOR_CREATING_EPICS~~~ + +## Rule #1 +Every epic must have only coding involved. There should never be a epic that is only testing or ensuring something works. There shouldn't be a epic for researching, deployment, writing documentation, testing or anything that is not writing the actual code. Testing if app works will be done as part of each epic. +Do not leave anything for interpretation, e.g. if something can be done in multiple ways, specify which way should be used and be as clear as possible. + +## Rule #2 +This rule applies to the number of epics you will create. +Every app should have different number of epics depending on complexity. Think epic by epic and create the minimum number of epics that are needed to develop this app. +Simple apps should have only 1 epic. More complex apps should have more epics. Do not create more epics than needed. + +## Rule #3 +This rule applies to writing epic 'description'. +Every epic must have a clear, high level, and short 1-2 sentence 'description'. It must be very clear so that even non technical users who are reviewing it and just moved to this project can understand what is goal for the epic. + +** MOST IMPORTANT RULES ** +## Rule #4 (MOST IMPORTANT RULE) +This rule applies to thinking about the API endpoints specified above between START_OF_FRONTEND_API_FILES and END_OF_FRONTEND_API_FILES. +Each epic must be related to one or more API endpoints that are called from the frontend files. Go through all API endpoints called from the frontend - if there are multiple endpoints related to a single entity (for example, CRUD operations on a database model), you can put them in the same epic but otherwise, make sure that API endpoints for different entities are in different epics. The epics you create **MUST** cover **ALL** API endpoints mentioned in the frontend files above. + +## Rule #5 (MOST IMPORTANT RULE) +This rule applies to order of epics. +Epics will be executed in same order that you output them. You must order them in a logical way so that epics that depend on other functionalities are implemented in later stage. The general order should be as follows: +{% if task_type == 'app' %} +- The first epic **MUST** be to implement authentication. In the first epic, the first task **MUST** be to remove the mocked data for making a register and login API requests from the frontend (in the client/src/api/auth.js file) - do **NOT** split this task into multiple tasks. Next task, if needed, should be to update the User model with more required parameters like different roles, different user data, etc. In the second task, make sure to add all necessary fields to the User model in the backend. +{% endif %} +- The {% if task_type == 'app' %}second{% else %}first{% endif %} epic **MUST** be to create any necessary scripts (if there are any). For example, a script to create an admin user, a script to seed the database, etc. +- {% if task_type == 'app' %}Finally{% else %}Then{% endif %}, all other epics must be about creating database models and CRUD operations (each epic must contain CRUD operations only for one single model - never for multiple). Pay attention to the API requests inside files in `client/api/` folder because they are currently using mocked data and whenever you implement an API endpoint, you just need to replace the mocked data with the real API request to the backend. + +## Rule #6 +Create epics for things that are not yet implemented. Do not reimplement what's already done. If something is already implemented, do not create epic for it. Continue from the implementation already there. + +~~~END_OF_RULES_FOR_CREATING_EPICS~~~ diff --git a/core/prompts/partials/relative_paths.prompt b/core/prompts/partials/relative_paths.prompt new file mode 100644 index 00000000..0fe2700e --- /dev/null +++ b/core/prompts/partials/relative_paths.prompt @@ -0,0 +1,4 @@ +IMPORTANT: Pay attention to file paths: if the command or argument is a file or folder from the project, use paths relative to the project root. + For example: + - use `dirname/filename.py` instead of `/path/to/project/dirname/filename.py` + - use `filename.js` instead of `./filename.js` diff --git a/core/prompts/partials/user_feedback.prompt b/core/prompts/partials/user_feedback.prompt new file mode 100644 index 00000000..f8926ad3 --- /dev/null +++ b/core/prompts/partials/user_feedback.prompt @@ -0,0 +1,15 @@ +{% if user_feedback %} +User who was using the app "{{ state.branch.project.name }}" sent you this feedback: +``` +{{ user_feedback }} +``` +{% endif %} +{% if user_feedback_qa %} +Feedback was not clear enough so you asked user for additional information and got this response: +``` +{% for row in user_feedback_qa %} +Q: {{ row.question }} +A: {{ row.answer }} +{% endfor %} +``` +{% endif %} \ No newline at end of file diff --git a/core/prompts/problem-solver/get_alternative_solutions.prompt b/core/prompts/problem-solver/get_alternative_solutions.prompt new file mode 100644 index 00000000..dbe41f2e --- /dev/null +++ b/core/prompts/problem-solver/get_alternative_solutions.prompt @@ -0,0 +1,61 @@ +You are working on an app called "{{ state.branch.project.name }}" and you need to write code for the entire {% if state.epics|length > 1 %}feature{% else %}app{% endif %} based on the tasks that the tech lead gives you. So that you understand better what you're working on, you're given other specs for "{{ state.branch.project.name }}" as well. + +Here is a high level description of "{{ state.branch.project.name }}": +``` +{{ state.specification.description }} +``` + +{% include "partials/features_list.prompt" %} + +We've broken the development of this {% if state.epics|length > 1 %}feature{% else %}app{% endif %} down to these tasks: +``` +{% for task in state.tasks %} +{{ loop.index }}. {{ task.description }}{% if task.get("status") == "done" %} (completed){% endif %} +{% endfor %} +``` + +{% if state.current_task %} +You are currently working on, and have to focus only on, this task: +``` +{{ state.current_task.description }} +``` + +{% endif %} +A part of the app is already finished. +{% include "partials/files_list.prompt" %} + +You are trying to solve an issue that your colleague is reporting. +{% if previous_solutions|length > 0 %} +You tried {{ previous_solutions|length }} times to solve it but it was unsuccessful. In last few attempts, your colleague gave you this report: +{% for solution in previous_solutions[-3:] %} +----------------------------start_of_report_{{ loop.index }}---------------------------- +{{ solution.user_feedback }} +----------------------------end_of_report_{{ loop.index }}---------------------------- + +Then, you gave the following proposal (proposal_{{ loop.index }}) of what needs to be done to fix the issue: +----------------------------start_of_proposal_{{ loop.index }}---------------------------- +{{ solution.description }} +----------------------------end_of_of_proposal_{{ loop.index }}---------------------------- + +{% if not loop.last %} +Then, upon implementing these changes, your colleague came back with the following report: +{% endif %} +{% endfor %} +{% endif %} + +{% if user_input != '' %} +Your colleague who is testing the app "{{ state.branch.project.name }}" sent you this report now: +``` +{{ user_input }} +``` + +You tried to solve this problem before but your colleague is telling you that you got into a loop where all your tries end up the same way - with an error. +{%- endif -%} + +It seems that the solutions you're proposing aren't working. + +Now, think step by step about 5 alternative solutions to get this code to work that are most probable to solve this issue. + +Every proposed solution needs to be concrete and not vague (eg, it cannot be "Review and change apps functionality") and based on the code changes. A solution can be complex if it's related to the same part of the code (eg. "Try changing the input variables X, Y and Z to a method N"). + +Order them in the order of the biggest probability of fixing the problem. A developer will then go through this list item by item, try to implement it, and check if it solved the issue until the end of the list. diff --git a/core/prompts/problem-solver/iteration.prompt b/core/prompts/problem-solver/iteration.prompt new file mode 100644 index 00000000..97d7d10c --- /dev/null +++ b/core/prompts/problem-solver/iteration.prompt @@ -0,0 +1 @@ +{% extends "troubleshooter/iteration.prompt" %} \ No newline at end of file diff --git a/core/prompts/problem-solver/system.prompt b/core/prompts/problem-solver/system.prompt new file mode 100644 index 00000000..e69de29b diff --git a/core/prompts/pythagora/commit.prompt b/core/prompts/pythagora/commit.prompt new file mode 100644 index 00000000..0e2f5825 --- /dev/null +++ b/core/prompts/pythagora/commit.prompt @@ -0,0 +1,7 @@ +You are working on an app called "{{ state.branch.project.name }}" and you need to generate commit message for next "git commit" command. + +Here are the changes that were made from last commit: +{{ git_diff }} + +Respond ONLY with the commit message that you would use for the next "git commit" command, nothing else. Do not use quotes, backticks or anything else, just plain text. +Commit message should be short and descriptive of the changes made since last commit. diff --git a/core/prompts/spec-writer/add_new_feature.prompt b/core/prompts/spec-writer/add_new_feature.prompt new file mode 100644 index 00000000..b9ce2599 --- /dev/null +++ b/core/prompts/spec-writer/add_new_feature.prompt @@ -0,0 +1,19 @@ +Your team has taken the client brief and turned it into a project specification. +Afterwards the client added a description for a new feature to be added to the project specification. +Your job is to update the project specification so that it contains the new feature information but does not lack any of the information from the original project specification. + +This might include: +* details on how the app should work +* information which 3rd party packages or APIs to use or avoid +* concrete examples of API requests/responses, library usage, or other external documentation + +Here is the original project specification: +{{ state.specification.description }} + +Here is the new feature description: +---FEATURE-DESCRIPTION-START--- +{{ feature_description }} +---FEATURE-DESCRIPTION-END--- + +In your response, output only the new updated project specification, without any additional messages to the user. +If there is no feature description just output the original project specification. diff --git a/core/prompts/spec-writer/ask_questions.prompt b/core/prompts/spec-writer/ask_questions.prompt new file mode 100644 index 00000000..c1335685 --- /dev/null +++ b/core/prompts/spec-writer/ask_questions.prompt @@ -0,0 +1,76 @@ +Your task is to talk to a new client and develop a detailed specification for a new application the client wants to build. This specification will serve as an input to an AI software developer and thus must be very detailed, contain all the project functionality and precisely define behaviour, 3rd-party integrations (if any), etc. + +The AI developer prefers working on web apps using Node/Express/MongoDB/Mongoose/EJS stack, and use vanilla JS with Bootstrap on the frontend, unless the client has different requirements. +Try to avoid the use of Docker, Kubernetes, microservices and single-page app frameworks like React, Next.js, Angular, Vue or Svelte unless the brief explicitly requires it. + +In your work, follow these important rules: +* In your communication with the client, be straightforward, concise, and focused on the task. +* Ask questions ONE BY ONE. This is very important, as the client is easily confused. If you were to ask multiple questions the user would probably miss some questions, so remember to always ask the questions one by one +* Ask specific questions, taking into account what you already know about the project. For example, don't ask "what features do you need?" or "describe your idea"; instead ask "what is the most important feature?" +* Pay special attention to any documentation or information that the project might require (such as accessing a custom API, etc). Be sure to ask the user to provide information and examples that the developers will need to build the proof-of-concept. You will need to output all of this in the final specification. +* This is a a prototype project, it is important to have small and well-defined scope. If the scope seems to grow too large (beyond a week or two of work for one developer), ask the user if they can simplify the project. +* Do not address non-functional requirements (performance, deployment, security, budget, timelines, etc...). We are only concerned with functional and technical specification here. +* Do not address deployment or hosting, including DevOps tasks to set up a CI/CD pipeline +* Don't address or invision any future development (post proof-of-concept), the scope of your task is to only spec the PoC/prototype. +* If the user provided specific information on how to access 3rd party API or how exactly to implement something, you MUST include that in the specification. Remember, the AI developer will only have access to the specification you write. + +Ensure that you have all the information about: +* overall description and goals for the app +* all the features of the application +* functional specification + * how the user will use the app + * enumerate all the parts of the application (eg. pages of the application, background processing if any, etc); for each part, explain *in detail* how it should work from the perspective of the user + * identify any constraints, business rules, user flows or other important info that affect how the application works or how it is used +* technical specification + * what kind of an application this is and what platform/technologies will be used + * the architecture of the application (what happens on backend, frontend, mobile, background tasks, integration with 3rd party services, etc) + * detailed description of each component of the application architecture +* integration specification + * any 3rd party apps, services, APIs that will be used (eg. for auth, payments, etc..) + * if a custom API is used, precise definitions, with examples, how to use the custom API or do the custom integration + +If you identify any missing information or need clarification on any vague or ambiguous parts of the brief, ask the client about it. + +Important note: don't ask trivial questions for obvious or unimportant parts of the app, for example: +* Bad questions example 1: + * Client brief: I want to build a hello world web app + * Bad questions: + * What title do you want for the web page that displays "Hello World"? + * What color and font size would you like for the "Hello World" text to be displayed in? + * Should the "Hello World" message be static text served directly from the server, or would you like it implemented via JavaScript on the client side? + * Explanation: There's no need to micromanage the developer(s) and designer(s), the client would've specified these details if they were important. + +If you ask such trivial questions, the client will think you're stupid and will leave. DON'T DO THAT + +Think carefully about what a developer must know to be able to build the app. The specification must address all of this information, otherwise the AI software developer will not be able to build the app. + +When you gather all the information from the client, output the complete specification. Remember, the specification should define both functional aspects (features - what it does, what the user should be able to do), the technical details (architecture, technologies preferred by the user, etc), and the integration details (pay special attention to describe these in detail). Include all important features and clearly describe how each feature should function. IMPORTANT: Do not add any preamble (eg. "Here's the specification....") or conclusion/commentary (eg. "Let me know if you have further questions")! + +Here's an EXAMPLE initial prompt: +---start-of-example-output--- +Online forum similar to Hacker News (news.ycombinator.com), with a simple and clean interface, where people can post links or text posts, and other people can upvote, downvote and comment on. Reading is open to anonymous users, but users must register to post, upvote, downvote or comment. Use simple username+password authentication. The forum should be implemented in Node.js with Express framework, using MongoDB and Mongoose ORM. + +The UI should use EJS view engine, Bootstrap for styling and plain vanilla JavaScript. Design should be simple and look like Hacker News, with a top bar for navigation, using a blue color scheme instead of the orange color in HN. The footer in each page should just be "Built using Pythagora". + +Each story has a title (one-line text), a link (optional, URL to an external article being shared on AI News), and text (text to show in the post). Link and text are mutually exclusive - if the submitter tries to use both, show them an error. + +Use the following algorithm to rank top stories, and comments within a story: "score = upvotes - downvotes + comments - sqrt(age)" , where "upvotes" and "downvotes" are the number of upvotes and downvotes the story or comment has, "comments" is the number of comments for a story (total), or the number of sub-comments (for a comment), and "age" is how old is the story, in minutes, and "sqrt" is the square root function. + +Implement the following pages: + +* / - shows the top 20 posted stories, ranked using the scoring algorithm, with a "More" link that shows the next 20 (pagination using "p" query parameter), and so on +* /newest - shows the latest 20 posted stories, ranked chronologically (newest first), with a "More" link that shows the next 20 (pagination using "p" query parameter), and so on +* /submit - shows a form to submit a new story, upon submitting the user should get redirected to /newest +* /login - shows a login form (username, password, "login" button, and a link to register page for new users) +* /register - shows a register form (username, password, "register" button, and a link to login page for existing users) +* /item - shows the story (use "id" query parameter to pass the story ID to this route) +* /comment - shows the form to send a comment (just a textarea and "submit" button) - upon commenting, the person should get redirected to the story they commented on + +The / and /newest pages should show the story title (link to the external article if "link" is set, otherwise link to the story item /item page), number of points (points = upvotes - downvotes), poster username (no link), how old is the story ("x minutes ago", "y hours ago" or "z days ago"), and "xyz comments" (link to /item page of the story). This is basically the same how HN shows it. + +The /item page should also follow the layout for HN in how it shows the story, and the comments tree. Instead of the embedded "reply" form, the story should just have a "comment" button that goes to the /comment page, similar to the "reply" link underneath each comment. Both should link to the /comment page. +---end-of-example-output--- + +Remember, this is important: the AI developer will not have access to client's initial description and transcript of your conversation. The developer will only see the specification you output on the end. It is very important that the spec captures *all* the details of the project in as much detail and precision as possible. + +Note: after the client reads the specification you create, the client might have additional comments or suggestions. In this case, continue the discussion with the user until you get all the new information and output the newly updated spec again. diff --git a/core/prompts/spec-writer/prompt_complexity.prompt b/core/prompts/spec-writer/prompt_complexity.prompt new file mode 100644 index 00000000..5ee09e20 --- /dev/null +++ b/core/prompts/spec-writer/prompt_complexity.prompt @@ -0,0 +1,15 @@ +{% if is_feature %} +Here is the app description that is fully built already: +``` +{{ state.specification.description }} +``` +Now I will show you the feature description that needs to be added to the app: +{% endif %} +``` +{{ prompt }} +``` + +{% if not is_feature %}The above is a user prompt for application/software tool they are trying to develop. {% endif %}Determine the complexity of the user's request. Do NOT respond with thoughts, reasoning, explanations or anything similar, return ONLY a string representation of the complexity level. Use the following scale: +"hard" for high complexity +"moderate" for moderate complexity +"simple" for low complexity diff --git a/core/prompts/spec-writer/review_spec.prompt b/core/prompts/spec-writer/review_spec.prompt new file mode 100644 index 00000000..043a340a --- /dev/null +++ b/core/prompts/spec-writer/review_spec.prompt @@ -0,0 +1,22 @@ +Your team has taken the client brief and turned it into a project specification. + +Your job is to check the specification and identify all the information that is contained in the client brief, but missing from the specification. + +This might include: +* details on how the app should work +* information which 3rd party packages or APIs to use or avoid +* concrete examples of API requests/responses, library usage, or other external documentation + +Here is the client brief: +---CLIENT-BRIEF-START--- +{{ desc }} +---CLIENT-BRIEF-END--- + +Here is the specification your team came up with: +---SPEC-START--- +{{ spec }} +---SPEC-END--- + +In your response, output all the information that is present in the client brief but missing from the spec, so it can be appended. + +Note: don't output suggestion to your team to take back to the drawing board. Instead, just output the missing information and the team will append it to the generated spec. If there is no missing information, just output an empty response (''). diff --git a/core/prompts/spec-writer/system.prompt b/core/prompts/spec-writer/system.prompt new file mode 100644 index 00000000..675c7cb6 --- /dev/null +++ b/core/prompts/spec-writer/system.prompt @@ -0,0 +1 @@ +You are a product owner working in a software development agency. diff --git a/core/prompts/tech-lead/epic_breakdown.prompt b/core/prompts/tech-lead/epic_breakdown.prompt new file mode 100644 index 00000000..14b54e2f --- /dev/null +++ b/core/prompts/tech-lead/epic_breakdown.prompt @@ -0,0 +1,19 @@ +Ok, great. Now, you need to take the epic #{{ epic_number }} ("{{ epic_description }}") and break it down into smaller tasks. Each task is one testable whole that the user can test and commit. Each task will be one commit that has to be testable by a human. Return the list of tasks for the Epic #{{ epic_number }}. For each task, write the task description and a description of how a human should test if the task is successfully implemented or not. Keep in mind that there can be 1 task or multiple, depending on the complexity of the epic. The epics will be implemented one by one so make sure that the user needs to be able to test each task you write - for example, if something will be implemented in the epics after the epic #{{ epic_number }}, then you cannot write it here because the user won't be able to test it. + +You need to specify tasks so that all these API endpoints get implemented completely. For each API endpoint that needs to be implemented, make sure to create a separate task so each task has only one API endpoint to implement. Also, you must not create tasks that don't have an endpoint that they are related to - for example, sometimes there is no "update" endpoint for a specific entity so you don't need to create a task for that. + +You can think of tasks as a unit of functionality that needs to have a frontend component and a backend component (don't split backend and frontend of the same functionality in separate tasks). + +**IMPORTANT: components of a single task** +When thinking about the scope of a single task, here are the components that need to be put into the same task: +1. The implementation of the backend API endpoint together with the frontend API request implementation (removing the mocked data and replacing it with the real API request) +2. The implementation of the database model +3. The utility function (eg. 3rd party integration) that is needed for this endpoint. + +**IMPORTANT: order of tasks** +The tasks you create **MUST** be in the order that they should be implemented. When CRUD operations need to be implemented, first implement the Create operation, then Read, Update, and Delete. + +{% if state.has_frontend() and not state.is_feature() and (state.options|default({})).get('auth', True) %} +**IMPORTANT** +If you are working on the Epic #1 that needs to implement the authentication system. The first task **MUST** be to remove the mocked data for authentication (register and login). After that, add any custom authentication requirements like different roles, different user data, etc. +{% endif %} diff --git a/core/prompts/tech-lead/filter_files.prompt b/core/prompts/tech-lead/filter_files.prompt new file mode 100644 index 00000000..3a4279a4 --- /dev/null +++ b/core/prompts/tech-lead/filter_files.prompt @@ -0,0 +1,2 @@ +{# This is the same template as for Developer's filter files #} +{% extends "developer/filter_files.prompt" %} \ No newline at end of file diff --git a/core/prompts/tech-lead/filter_files_loop.prompt b/core/prompts/tech-lead/filter_files_loop.prompt new file mode 100644 index 00000000..6689b003 --- /dev/null +++ b/core/prompts/tech-lead/filter_files_loop.prompt @@ -0,0 +1,2 @@ +{# This is the same template as for Developer's filter files #} +{% extends "developer/filter_files_loop.prompt" %} \ No newline at end of file diff --git a/core/prompts/tech-lead/plan.prompt b/core/prompts/tech-lead/plan.prompt new file mode 100644 index 00000000..4775b5e4 --- /dev/null +++ b/core/prompts/tech-lead/plan.prompt @@ -0,0 +1,33 @@ +You are working in a software development agency and a project manager and software architect approach you telling you that you're assigned to {% if task_type == 'feature' %}add new feature to an existing project{% else %}work on a new project{% endif %}. +You are working on an app called "{{ state.branch.project.name }}" and you need to create a detailed development plan so that developers can start developing the app. + +{% include "partials/project_details.prompt" %} +{% include "partials/features_list.prompt" %} +{% if existing_summary %} + +The developers have already used a project scaffolding tool that creates the initial boilerplate for the project: +{{ existing_summary }} +{% endif %} + +{% include "partials/files_list.prompt" %} + +{% if task_type == 'feature' %} +Finally, here is the description of new feature that needs to be added to the app "{{ state.branch.project.name }}": +``` +{{ epic.description }} +``` +{% endif %} + +{% if epic.complexity and epic.complexity == 'simple' %} +This is very low complexity {{ task_type }} and because of that, you have to create ONLY one task that is sufficient to fully implement it. +{% else %} +Before we go into the coding part, your job is to split the development process of building the backend for this app into epics. Above, you can see a part of the backend that's already built and the files from the frontend that make requests to the backend. The rest of the frontend is built but is not shown above because it is not necessary for you to create a list of epics. +Now, based on the project details provided{% if task_type == 'feature' %} and new feature description{% endif %}, think epic by epic and create the entire development plan{% if task_type == 'feature' %} for new feature{% elif task_type == 'app' %}. {% if state.files %}Continue from the existing code listed above{% else %}Start from the project setup{% endif %} and specify each epic until the moment when the entire app should be fully working{% if state.files %}. IMPORTANT: You should not reimplement what's already done - just continue from the implementation already there.{% endif %}{% endif %} + +IMPORTANT! +Frontend is already built and you don't need to create epics for it. You only need to create epics for backend implementation and connect it to existing frontend. Keep in mind that some backend functionality is already implemented.{% if task_type == 'app' and (state.options|default({})).get('auth', True) %} The first epic **MUST** be to implement the authentication system.{% endif %} + +Strictly follow these rules: + +{% include "partials/project_tasks.prompt" %} +{% endif %} diff --git a/core/prompts/tech-lead/system.prompt b/core/prompts/tech-lead/system.prompt new file mode 100644 index 00000000..8cdc46e9 --- /dev/null +++ b/core/prompts/tech-lead/system.prompt @@ -0,0 +1,3 @@ +You are an experienced tech lead in a software development agency. +Your main job is to break down the project into epics that developers will do. +You must specify each epic as clear as possible. diff --git a/core/prompts/tech-writer/create_readme.prompt b/core/prompts/tech-writer/create_readme.prompt new file mode 100644 index 00000000..4d9e8d95 --- /dev/null +++ b/core/prompts/tech-writer/create_readme.prompt @@ -0,0 +1,37 @@ +You are working on a project called "{{ state.branch.project.name }}" and you need to create a detailed documentation for current state of project. Your first task is to create README.md file. + +{% include "partials/project_details.prompt" %} +{% include "partials/features_list.prompt" %} +{% include "partials/files_list.prompt" %} + +DO NOT specify commands to create any folders or files, they will be created automatically - just specify the relative path to file that needs to be written. + +{% include "partials/relative_paths.prompt" %} + +Now, based on the project details provided, think step by step and create README.md file for this project. The file should have the following format: + +# Project name + +Short description (a few sentences) of the project based on the project details. + +## Overview + +Description of the architecture and technologies used in the project, and the project structure. + +## Features + +Description of what the app can do and how it can be used. + +## Getting started + +### Requirements + +Required technologies/setup needed on the computer to run the project. + +### Quickstart + +How to set up the project and run it + +### License + +The project is proprietary (not open source), just output the standard Copyright (c) 2024. template here. diff --git a/core/prompts/tech-writer/system.prompt b/core/prompts/tech-writer/system.prompt new file mode 100644 index 00000000..3bf0e485 --- /dev/null +++ b/core/prompts/tech-writer/system.prompt @@ -0,0 +1 @@ +You are technical writer and as such, you excel in clear, concise communication, skillfully breaking down complex technical concepts for a variety of audiences. Your proficiency in research and attention to detail ensures accuracy and consistency in your work. You adeptly organize complex information in a user-friendly manner, understanding and anticipating the needs of your audience. Your collaborative skills enhance your ability to work effectively with diverse teams. In your role, you not only create documentation but also efficiently manage documentation projects, always prioritizing clarity and usefulness for the end-user. diff --git a/core/prompts/troubleshooter/breakdown.prompt b/core/prompts/troubleshooter/breakdown.prompt new file mode 100644 index 00000000..6ca24015 --- /dev/null +++ b/core/prompts/troubleshooter/breakdown.prompt @@ -0,0 +1,2 @@ +{# This is the same template as for Developer's breakdown because Troubleshooter is reusing it in a conversation #} +{% extends "developer/breakdown.prompt" %} diff --git a/core/prompts/troubleshooter/bug_report.prompt b/core/prompts/troubleshooter/bug_report.prompt new file mode 100644 index 00000000..ac273572 --- /dev/null +++ b/core/prompts/troubleshooter/bug_report.prompt @@ -0,0 +1,43 @@ +You're working on an new app and the user has just been testing it. + +{% include "partials/project_details.prompt" %} +{% include "partials/files_list.prompt" %} + +{% if user_instructions %} +The user was given instructions on how to test if the app is working correctly. Here are the instructions: +``` +{{ user_instructions }} +``` +{% endif %} + +The user then wrote this feedback: +``` +{{ user_feedback }} +``` +{% if additional_qa|length > 0 %} +Here are questions and answers that you already asked the user: +``` +{% for row in additional_qa %} +Q: {{ row.question }} +A: {{ row.answer }} +{% endfor %} +``` +{% endif %} + +Your job is to identify if feedback is good enough for you to solve the problem. If not, what information you need to solve the problem. Ask for any information that you need to solve the problem. +If you have enough information don't ask any questions. + +When thinking of questions, consider the following: +- After getting answers to your questions, you must be able to solve the problem. +- Ask only crucial questions. Do not ask for information that you do not need to solve the problem. +- Ask least amount of questions to get the most information and to solve the problem. +- Ask only questions from the list provided bellow. +- Ask questions in same order as they are in the list. +- Never repeat same question. + +Here is the list of questions you can ask: +"Can you please provide more information on what exactly you mean?" +"Can you please provide logs from the frontend?" +"Can you please provide logs from the backend?" +"What is the expected behavior and what is current behaviour?" +"On what page does the issue happen?" \ No newline at end of file diff --git a/core/prompts/troubleshooter/define_user_review_goal.prompt b/core/prompts/troubleshooter/define_user_review_goal.prompt new file mode 100644 index 00000000..44c068ea --- /dev/null +++ b/core/prompts/troubleshooter/define_user_review_goal.prompt @@ -0,0 +1,76 @@ +How can a human user test if this task was completed successfully? + +Please list actions, step by step, in order, that the user should take to verify the task. After each action, describe what the expected response is. + +**IMPORTANT** + +Follow these important rules when compiling a list of actions the user will take: + +1. Actions must be as specific as possible. You don't want the user to have to think anything through but rather that they just follow your instructions. +2. In case this task can be tested by making an API request, you should always prefer to test functionality in the browser. In case you can't do that, do not suggest how can a request be made with Postman but rather write a full cURL command that the user can just run. +3. Do not require the user to write any code or edit files to test this task. +4. If the user must run a command, assume the user already has a terminal opened in the project root directory (no need to instruct the user "open the terminal" or "make sure you're in the project directory") +5. The user is using {{ os }}, so the commands must run on that operating system +6. Assume system services, such as the database, are already set up and running. Don't ask user to install or run any software other than the app they're testing. +7. Don't ask the user to test things which aren't implemented yet (eg. opening a theoretical web page that doesn't exist yet, or clicking on a button that isn't implemented yet) +8. Think about if there is something that user needs to do manually to make the next testing step work +9. The human has the option to press the "Start App" button so don't instruct them to start the app in any way. +10. If the user needs to run a database command, make sure to specify the entire command that needs to be run + +Remember, these rules are very important and you must follow them! + +Here is an example output with a few user steps: +---example--- +{ + "steps": [ + { + "title": "Submit the form", + "action": "Open your web browser and visit 'http://localhost:5173/form'. Click on the 'Submit' button in the web form", + "result": "Form is submitted, the page is reloaded, and 'Thank you' message is shown", + }, + { + "title": "Check email", + "action": "Check your email inbox for the magic link. Click on the magic link to log in.", + "result": "You should be redirected back to the home page. The login status should now display 'Logged in as [your email]' and the 'Logout' button should be visible.", + }, + { + "title": "Log out", + "action": "Click on the 'Logout' button", + "result": "You should be redirected back to the home page. You should not be able to access the form and the 'Login' button should be visible.", + } + ] +} +---end_of_example--- + +If nothing needs to be tested for this task, instead of outputting the steps, just output an empty list like this: +---example_when_test_not_needed--- +{ + "steps": [] +} +---end_of_example_when_test_not_needed--- + +When you think about the testing instructions for the human, keep in mind the tasks that have been already implemented, the task that the human needs to test right now, and the tasks that are still not implemented. If something is not implemented yet, the user will not be able to test a functionality related to that. For example, if a task is to implement a feature that enables the user to create a company record and if you see that the feature to retrieve a list of company records is still not implemented, you cannot tell the human to open the page for viewing company records because it's still not implemented. In this example, you should tell the human to look into a database or some other way that they can verify if the company records are created. The current situation is like this: +Here are the tasks that are implemented: +``` +{% for task in state.tasks %} +{% if loop.index - 1 < current_task_index %} +{{ loop.index }}. {{ task.description }}{% if task.get("status") == "done" %} (completed){% endif %} + +{% endif %}{% endfor %} +``` + +Here is the task that the human needs to test: +``` +{{ current_task_index + 1 }}. {{ task.description }} +``` + +And here are the tasks that are still NOT implemented: +``` +{% for task in state.tasks %} +{% if loop.index - 1 > current_task_index %} +{{ loop.index }}. {{ task.description }}{% if task.get("status") == "done" %} (completed){% endif %} + +{% endif %}{% endfor %} +``` + +Knowing these rules, tell me, please list actions, step by step, in order, that the user should take to verify the task. diff --git a/core/prompts/troubleshooter/filter_files.prompt b/core/prompts/troubleshooter/filter_files.prompt new file mode 100644 index 00000000..3a4279a4 --- /dev/null +++ b/core/prompts/troubleshooter/filter_files.prompt @@ -0,0 +1,2 @@ +{# This is the same template as for Developer's filter files #} +{% extends "developer/filter_files.prompt" %} \ No newline at end of file diff --git a/core/prompts/troubleshooter/filter_files_loop.prompt b/core/prompts/troubleshooter/filter_files_loop.prompt new file mode 100644 index 00000000..6689b003 --- /dev/null +++ b/core/prompts/troubleshooter/filter_files_loop.prompt @@ -0,0 +1,2 @@ +{# This is the same template as for Developer's filter files #} +{% extends "developer/filter_files_loop.prompt" %} \ No newline at end of file diff --git a/core/prompts/troubleshooter/get_route_files.prompt b/core/prompts/troubleshooter/get_route_files.prompt new file mode 100644 index 00000000..45715d8f --- /dev/null +++ b/core/prompts/troubleshooter/get_route_files.prompt @@ -0,0 +1,13 @@ +{# eval-tool test ID: 74918173-59e4-4005-bf19-d28f3bc9f06c + +This was added in June 2024, to improve the accuracy of user testing instructions. +We've noticed that the LLM would often times include incorrect URLs in the user testing +instructions, because it wasn't aware of the routes used in the application. +The solution was to add the entire content of all the files that have routes defined in +them, and this prompt selects those files. + +#} +{% include "partials/project_details.prompt" %} +{% include "partials/files_list.prompt" %} + +Your task is to identify all the files, from the above list, that have routes defined in them. Return just the file paths as a JSON list named "files", DO NOT return anything else. If there are no files with routes, return an empty list. diff --git a/core/prompts/troubleshooter/get_run_command.prompt b/core/prompts/troubleshooter/get_run_command.prompt new file mode 100644 index 00000000..522f6075 --- /dev/null +++ b/core/prompts/troubleshooter/get_run_command.prompt @@ -0,0 +1,9 @@ +How can I run this app? +**IMPORTANT** +Do not reply with anything else but the command with which I can run this app with. +For example, if the command is "python app.py", then your response needs to be only `python app.py` without the ` + +{% include "partials/relative_paths.prompt" %} + +If there is no command to run reply with empty response. +For example, if we only setup package.json and no other files are coded there is no command to run so respond with `` without the ` diff --git a/core/prompts/troubleshooter/iteration.prompt b/core/prompts/troubleshooter/iteration.prompt new file mode 100644 index 00000000..fc5e9de5 --- /dev/null +++ b/core/prompts/troubleshooter/iteration.prompt @@ -0,0 +1,55 @@ +You are working on an app called "{{ state.branch.project.name }}" and you need to write code for the entire application. + +{% include "partials/project_details.prompt" %} + +{% if state.tasks and state.current_task %} +Development process of this app was split into smaller tasks. Here is the list of all tasks: +``` +{% for task in state.tasks %} +{{ loop.index }}. {{ task.description }} +{% endfor %} +``` + +You are currently working on, and have to focus only on, this task: +``` +{{ state.current_task.description }} +``` + +{% endif %} +A part of the app is already finished. +{% include "partials/files_list.prompt" %} + +{% include "partials/user_feedback.prompt" %} + +{% if test_instructions %} +User was testing the current implementation of the app when they requested some changes to the app. These are the testing instructions: +``` +{% for step in test_instructions %} +Step #{{ loop.index }} +Action: {{ step.action }} +Expected result: {{ step.result }} +{% endfor %} +``` +{% endif %} + +{% if next_solution_to_try is not none %} +Focus on solving this issue in the following way: +``` +{{ next_solution_to_try }} +``` +{% endif %} +{% include "partials/doc_snippets.prompt" %} +Now, tell me how can we implement the changes that the user requested. Think step by step and explain each change you want to make and write code snippets that you want to change. + +**IMPORTANT** +Think about all information provided. Your job is to look at big picture by analysing all files to find where the issue is. +When there are multiple things that have to be done in one file write everything as one step and don't split it in multiple steps. +You can count that the environment is set up previously and packages listed in files are installed so tell me only commands needed for installation of new dependencies, if there are any. + +**IMPORTANT** +If report mentions *multiple* issues, treat it as if you got several separate reports: think through each, and provide solutions for each. + +{% include "partials/execution_order.prompt" %} +{% include "partials/file_size_limit.prompt" %} +{% include "partials/file_naming.prompt" %} +{% include "partials/relative_paths.prompt" %} diff --git a/core/prompts/troubleshooter/system.prompt b/core/prompts/troubleshooter/system.prompt new file mode 100644 index 00000000..e69de29b diff --git a/core/state/__init__.py b/core/state/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/core/state/state_manager.py b/core/state/state_manager.py new file mode 100644 index 00000000..f40a8fd2 --- /dev/null +++ b/core/state/state_manager.py @@ -0,0 +1,758 @@ +import asyncio +import os.path +import traceback +from contextlib import asynccontextmanager +from typing import TYPE_CHECKING, Optional +from uuid import UUID, uuid4 + +from sqlalchemy import inspect, select +from tenacity import retry, stop_after_attempt, wait_fixed + +from core.config import FileSystemType, get_config +from core.db.models import Branch, ExecLog, File, FileContent, LLMRequest, Project, ProjectState, UserInput +from core.db.models.specification import Complexity, Specification +from core.db.session import SessionManager +from core.disk.ignore import IgnoreMatcher +from core.disk.vfs import LocalDiskVFS, MemoryVFS, VirtualFileSystem +from core.llm.request_log import LLMRequestLog, LLMRequestStatus +from core.log import get_logger +from core.proc.exec_log import ExecLog as ExecLogData +from core.telemetry import telemetry +from core.ui.base import UIBase +from core.ui.base import UserInput as UserInputData + +if TYPE_CHECKING: + from core.agents.base import BaseAgent + +log = get_logger(__name__) + + +class StateManager: + """ + Manages loading, updating and saving project states. + + All project state references reading the current state + should use `StateManager.current` attribute. All changes + to the state should be done through the `StateManager.next` + attribute. + """ + + current_state: Optional[ProjectState] + next_state: Optional[ProjectState] + + def __init__(self, session_manager: SessionManager, ui: Optional[UIBase] = None): + self.session_manager = session_manager + self.ui = ui + self.file_system = None + self.project = None + self.branch = None + self.current_state = None + self.next_state = None + self.current_session = None + self.blockDb = False + self.git_available = False + self.git_used = False + self.options = {} + + @asynccontextmanager + async def db_blocker(self): + while self.blockDb: + await asyncio.sleep(0.1) # Wait if blocked + + try: + self.blockDb = True # Set the block + yield + finally: + self.blockDb = False # Unset the block + + async def list_projects(self) -> list[Project]: + """ + List projects with branches + + :return: List of projects with all their branches. + """ + async with self.session_manager as session: + return await Project.get_all_projects(session) + + async def create_project(self, name: str, folder_name: Optional[str] = None) -> Project: + """ + Create a new project and set it as the current one. + + :param name: Project name. + :return: The Project object. + """ + session = await self.session_manager.start() + project = Project(name=name, folder_name=folder_name) + branch = Branch(project=project) + state = ProjectState.create_initial_state(branch) + session.add(project) + + # This is needed as we have some behavior that traverses the files + # even for a new project, eg. offline changes check and stats updating + await state.awaitable_attrs.files + + await session.commit() + + log.info( + f'Created new project "{name}" (id={project.id}) ' + f'with default branch "{branch.name}" (id={branch.id}) ' + f"and initial state id={state.id} (step_index={state.step_index})" + ) + await telemetry.trace_code_event("create-project", {"name": name}) + + self.current_session = session + self.current_state = state + self.next_state = state + self.project = project + self.branch = branch + self.file_system = await self.init_file_system(load_existing=False) + return project + + async def delete_project(self, project_id: UUID) -> bool: + session = await self.session_manager.start() + rows = await Project.delete_by_id(session, project_id) + if rows > 0: + await Specification.delete_orphans(session) + await FileContent.delete_orphans(session) + + await session.commit() + + if rows > 0: + log.info(f"Deleted project {project_id}.") + return bool(rows) + + async def load_project( + self, + *, + project_id: Optional[UUID] = None, + branch_id: Optional[UUID] = None, + step_index: Optional[int] = None, + ) -> Optional[ProjectState]: + """ + Load project state from the database. + + If `branch_id` is provided, load the latest state of the branch. + Otherwise, if `project_id` is provided, load the latest state of + the `main` branch in the project. + + If `step_index' is provided, load the state at the given step + of the branch instead of the last one. + + The returned ProjectState will have branch and branch.project + relationships preloaded. All other relationships must be + explicitly loaded using ProjectState.awaitable_attrs or + AsyncSession.refresh. + + :param project_id: Project ID (keyword-only, optional). + :param branch_id: Branch ID (keyword-only, optional). + :param step_index: Step index within the branch (keyword-only, optional). + :return: The ProjectState object if found, None otherwise. + """ + + if self.current_session: + log.info("Current session exists, rolling back changes.") + await self.rollback() + + state = None + session = await self.session_manager.start() + + if branch_id is not None: + branch = await Branch.get_by_id(session, branch_id) + if branch is not None: + if step_index: + state = await branch.get_state_at_step(step_index) + else: + state = await branch.get_last_state() + + elif project_id is not None: + project = await Project.get_by_id(session, project_id) + if project is not None: + branch = await project.get_branch() + if branch is not None: + if step_index: + state = await branch.get_state_at_step(step_index) + else: + state = await branch.get_last_state() + else: + raise ValueError("Project or branch ID must be provided.") + + if state is None: + await self.session_manager.close() + log.debug( + f"Unable to load project state (project_id={project_id}, branch_id={branch_id}, step_index={step_index})" + ) + return None + + # TODO: in the future, we might want to create a new branch here? + await state.delete_after() + await session.commit() + + self.current_session = session + self.current_state = state + self.branch = state.branch + self.project = state.branch.project + self.next_state = await state.create_next_state() + self.file_system = await self.init_file_system(load_existing=True) + log.debug( + f"Loaded project {self.project} ({self.project.id}) " + f"branch {self.branch} ({self.branch.id}" + f"step {state.step_index} (state id={state.id})" + ) + + if self.current_state.current_epic and self.current_state.current_task and self.ui: + await self.ui.send_epics_and_tasks( + self.current_state.current_epic.get("sub_epics"), + self.current_state.tasks, + ) + source = self.current_state.current_epic.get("source", "app") + await self.ui.send_task_progress( + self.current_state.tasks.index(self.current_state.current_task) + 1, + len(self.current_state.tasks), + self.current_state.current_task["description"], + source, + "in-progress", + self.current_state.get_source_index(source), + self.current_state.tasks, + ) + + telemetry.set( + "architecture", + { + "system_dependencies": self.current_state.specification.system_dependencies, + "package_dependencies": self.current_state.specification.package_dependencies, + }, + ) + telemetry.set("example_project", self.current_state.specification.example_project) + telemetry.set("is_complex_app", self.current_state.specification.complexity != Complexity.SIMPLE) + telemetry.set("templates", self.current_state.specification.templates) + + return self.current_state + + @retry(stop=stop_after_attempt(3), wait=wait_fixed(1)) + async def commit_with_retry(self): + try: + await self.current_session.commit() + except Exception as e: + log.error(f"Commit failed: {str(e)}") + raise + + async def commit(self) -> ProjectState: + """ + Commit the new project state to the database. + + This commits `next_state` to the database, making the changes + permanent, then creates a new state for further changes. + + :return: The committed state. + """ + try: + if self.next_state is None: + raise ValueError("No state to commit.") + if self.current_session is None: + raise ValueError("No database session open.") + + log.debug("Committing session") + await self.commit_with_retry() + log.debug("Session committed successfully") + + # Having a shorter-lived sessions is considered a good practice in SQLAlchemy, + # so we close and recreate the session for each state. This uses db + # connection from a connection pool, so it is fast. Note that SQLite uses + # no connection pool by default because it's all in-process so it's fast anyway. + self.current_session.expunge_all() + await self.session_manager.close() + self.current_session = await self.session_manager.start() + + self.current_state = self.next_state + self.current_session.add(self.next_state) + self.next_state = await self.current_state.create_next_state() + + # After the next_state becomes the current_state, we need to load + # the FileContent model, which was previously loaded by the load_project(), + # but is not populated by the `create_next_state()` + for f in self.current_state.files: + await f.awaitable_attrs.content + + telemetry.inc("num_steps") + + # FIXME: write a test to verify files (and file content) are preloaded + return self.current_state + + except Exception as e: + log.error(f"Error during commit: {str(e)}") + log.error(traceback.format_exc()) + raise + + async def rollback(self): + """ + Abandon (rollback) the next state changes. + """ + if not self.current_session: + return + await self.current_session.rollback() + await self.session_manager.close() + self.current_session = None + return + + async def log_llm_request(self, request_log: LLMRequestLog, agent: Optional["BaseAgent"] = None): + """ + Log the request to the next state. + + Note: contrary to most other methods, this stores the information + to the CURRENT state, not the next one. As the requests/responses + depend on the current state, it makes it easier to analyze the + database by just looking at a single project state later. + + :param request_log: The request log to log. + """ + async with self.db_blocker(): + try: + telemetry.record_llm_request( + request_log.prompt_tokens + request_log.completion_tokens, + request_log.duration, + request_log.status != LLMRequestStatus.SUCCESS, + ) + LLMRequest.from_request_log(self.current_state, agent, request_log) + + except Exception as e: + if self.ui: + await self.ui.send_message(f"An error occurred: {e}") + + async def log_user_input(self, question: str, response: UserInputData): + """ + Log the user input to the current state. + + Note: contrary to most other methods, this stores the information + to the CURRENT state, not the next one. As the user interactions + depend on the current state, it makes it easier to analyze the + database by just looking at a single project state later. + + :param question: The question asked. + :param response: The user response. + """ + telemetry.inc("num_inputs") + UserInput.from_user_input(self.current_state, question, response) + + async def log_command_run(self, exec_log: ExecLogData): + """ + Log the command run to the current state. + + Note: contrary to most other methods, this stores the information + to the CURRENT state, not the next one. As the command execution + depend on the current state, it makes it easier to analyze the + database by just looking at a single project state later. + + :param exec_log: The command execution log. + """ + telemetry.inc("num_commands") + ExecLog.from_exec_log(self.current_state, exec_log) + + async def log_event(self, type: str, **kwargs): + """ + Log an event like: + + + * start of epic + * start of task + * start of iteration + * end of task + * end of epic + * loop detected + """ + # TODO: implement this + # Consider seting this/orchestrator so that the side effect is to send + # the update to the UI (vscode extension) + + async def log_task_completed(self): + telemetry.inc("num_tasks") + if not self.next_state.unfinished_tasks: + if len(self.current_state.epics) == 1: + telemetry.set("end_result", "success:frontend") + elif len(self.current_state.epics) == 2: + telemetry.set("end_result", "success:initial-project") + else: + telemetry.set("end_result", "success:feature") + await telemetry.send() + + async def get_file_by_path(self, path: str) -> Optional[File]: + """ + Get a file from the current project state, by the file path. + + :param path: The file path. + :return: The file object, or None if not found. + """ + return self.current_state.get_file_by_path(path) + + async def save_file( + self, + path: str, + content: str, + metadata: Optional[dict] = None, + from_template: bool = False, + ): + """ + Save a file to the project. + + Note that the file is saved to the file system immediately, but in + database it may be rolled back if `next_state` is never committed. + + :param path: The file path. + :param content: The file content. + :param metadata: Optional metadata (eg. description) to save with the file. + :param from_template: Whether the file is part of a template. + """ + try: + original_content = self.file_system.read(path) + except ValueError: + original_content = "" + + # FIXME: VFS methods should probably be async + self.file_system.save(path, content) + + hash = self.file_system.hash_string(content) + async with self.db_blocker(): + file_content = await FileContent.store(self.current_session, hash, content) + + file = self.next_state.save_file(path, file_content) + # if self.ui and not from_template: + # await self.ui.open_editor(self.file_system.get_full_path(path)) + if metadata: + file.meta = metadata + + if not from_template: + delta_lines = len(content.splitlines()) - len(original_content.splitlines()) + telemetry.inc("created_lines", delta_lines) + + async def init_file_system(self, load_existing: bool) -> VirtualFileSystem: + """ + Initialize file system interface for the new or loaded project. + + When creating a new project, `load_existing` should be False to ensure a + new unique project folder is created. When loading an existing project, + `load_existing` should be True to allow using already-existing folder + with the project files. If the folder doesn't exist, it will be created. + + This also initializes the ignore mechanism, so that files are correctly + ignored as configured. + + :param load_existing: Whether to load existing files from the file system. + :return: The file system interface. + """ + config = get_config() + + if config.fs.type == FileSystemType.MEMORY: + return MemoryVFS() + + if config.fs.type != FileSystemType.LOCAL: + raise ValueError(f"Unsupported file system type: {config.fs.type}") + + while True: + root = self.get_full_project_root() + ignore_matcher = IgnoreMatcher( + root, + config.fs.ignore_paths, + ignore_size_threshold=config.fs.ignore_size_threshold, + ) + + try: + return LocalDiskVFS(root, allow_existing=load_existing, ignore_matcher=ignore_matcher) + except FileExistsError: + self.project.folder_name = self.project.folder_name + "-" + uuid4().hex[:7] + log.warning(f"Directory {root} already exists, changing project folder to {self.project.folder_name}") + await self.current_session.commit() + + def get_full_project_root(self) -> str: + """ + Get the full path to the project root folder. + + :return: The full path to the project root folder. + """ + config = get_config() + + if self.project is None: + raise ValueError("No project loaded") + return os.path.join(config.fs.workspace_root, self.project.folder_name) + + async def import_files(self) -> tuple[list[File], list[File]]: + """ + Scan the file system, import new/modified files, delete removed files. + + The files are saved to / removed from `next_state`, but not committed + to database until the new state is committed. + + :return: Tuple with the list of imported files and the list of removed files. + """ + known_files = {file.path: file for file in self.current_state.files} + files_in_workspace = set() + imported_files = [] + removed_files = [] + + for path in self.file_system.list(): + files_in_workspace.add(path) + content = self.file_system.read(path) + saved_file = known_files.get(path) + + if saved_file and saved_file.content.content == content: + continue + + # TODO: unify this with self.save_file() / refactor that whole bit + hash = self.file_system.hash_string(content) + log.debug(f"Importing file {path} (hash={hash}, size={len(content)} bytes)") + file_content = await FileContent.store(self.current_session, hash, content) + file = self.next_state.save_file(path, file_content, external=True) + imported_files.append(file) + + for path, file in known_files.items(): + if path not in files_in_workspace: + log.debug(f"File {path} was removed from workspace, deleting from project") + next_state_file = self.next_state.get_file_by_path(path) + self.next_state.files.remove(next_state_file) + removed_files.append(file.path) + + return imported_files, removed_files + + async def restore_files(self) -> list[File]: + """ + Restore files from the database to VFS. + + Warning: this could overwrite user's files on disk! + + :return: List of restored files. + """ + known_files = {file.path: file for file in self.current_state.files} + files_in_workspace = self.file_system.list() + + for disk_f in files_in_workspace: + if disk_f not in known_files: + self.file_system.remove(disk_f) + + restored_files = [] + for path, file in known_files.items(): + restored_files.append(file) + self.file_system.save(path, file.content.content) + + return restored_files + + async def get_modified_files(self) -> list[str]: + """ + Return a list of new or modified files from the file system. + + :return: List of paths for new or modified files. + """ + + modified_files = [] + files_in_workspace = self.file_system.list() + for path in files_in_workspace: + content = self.file_system.read(path) + saved_file = self.current_state.get_file_by_path(path) + if saved_file and saved_file.content.content == content: + continue + modified_files.append(path) + + # Handle files removed from disk + await self.current_state.awaitable_attrs.files + for db_file in self.current_state.files: + if db_file.path not in files_in_workspace: + modified_files.append(db_file.path) + + return modified_files + + async def get_modified_files_with_content(self) -> list[dict]: + """ + Return a list of new or modified files from the file system, + including their paths, old content, and new content. + + :return: List of dictionaries containing paths, old content, + and new content for new or modified files. + """ + + modified_files = [] + files_in_workspace = self.file_system.list() + + for path in files_in_workspace: + content = self.file_system.read(path) + saved_file = self.current_state.get_file_by_path(path) + + # If there's a saved file, serialize its content; otherwise, set it to None + saved_file_content = saved_file.content.content if saved_file else None + + if saved_file_content == content: + continue + + modified_files.append( + { + "path": path, + "file_old": saved_file_content, # Serialized content + "file_new": content, + } + ) + + # Handle files removed from disk + await self.current_state.awaitable_attrs.files + for db_file in self.current_state.files: + if db_file.path not in files_in_workspace: + modified_files.append( + { + "path": db_file.path, + "file_old": db_file.content.content, # Serialized content + "file_new": "", # Empty string as the file is removed + } + ) + + return modified_files + + def workspace_is_empty(self) -> bool: + """ + Returns whether the workspace has any files in them or is empty. + """ + return not bool(self.file_system.list()) + + def get_implemented_pages(self) -> list[str]: + """ + Get the list of implemented pages. + + :return: List of implemented pages. + """ + # TODO - use self.current_state plus response from the FE iteration + page_files = [file.path for file in self.next_state.files if "client/src/pages" in file.path] + return page_files + + async def update_implemented_pages_and_apis(self): + modified = False + pages = self.get_implemented_pages() + apis = await self.get_apis() + + # Get the current state of pages and apis from knowledge_base + current_pages = self.next_state.knowledge_base.get("pages", None) + current_apis = self.next_state.knowledge_base.get("apis", None) + + # Check if pages or apis have changed + if pages != current_pages or apis != current_apis: + modified = True + + if modified: + self.next_state.knowledge_base["pages"] = pages + self.next_state.knowledge_base["apis"] = apis + self.next_state.flag_knowledge_base_as_modified() + await self.ui.knowledge_base_update(self.next_state.knowledge_base) + + async def update_utility_functions(self, utility_function: dict): + """ + Update the knowledge base with the utility function. + + :param utility_function: Utility function to update. + """ + matched = False + for kb_util_func in self.next_state.knowledge_base.get("utility_functions", []): + if ( + utility_function["function_name"] == kb_util_func["function_name"] + and utility_function["file"] == kb_util_func["file"] + ): + kb_util_func["return_value"] = utility_function["return_value"] + kb_util_func["input_value"] = utility_function["input_value"] + kb_util_func["status"] = utility_function["status"] + matched = True + self.next_state.flag_knowledge_base_as_modified() + break + + if not matched: + if "utility_functions" not in self.next_state.knowledge_base: + self.next_state.knowledge_base["utility_functions"] = [] + self.next_state.knowledge_base["utility_functions"].append(utility_function) + + self.next_state.flag_knowledge_base_as_modified() + await self.ui.knowledge_base_update(self.next_state.knowledge_base) + + async def get_apis(self) -> list[dict]: + """ + Get the list of APIs. + + :return: List of APIs. + """ + apis = [] + for file in self.next_state.files: + if "client/src/api" not in file.path: + continue + session = inspect(file).async_session + result = await session.execute(select(FileContent).where(FileContent.id == file.content_id)) + file_content = result.scalar_one_or_none() + content = file_content.content + lines = content.splitlines() + for i, line in enumerate(lines): + if "// Description:" in line: + # TODO: Make this better!!! + description = line.split("Description:")[1] + endpoint = lines[i + 1].split("Endpoint:")[1] if len(lines[i + 1].split("Endpoint:")) > 1 else "" + request = lines[i + 2].split("Request:")[1] if len(lines[i + 2].split("Request:")) > 1 else "" + response = lines[i + 3].split("Response:")[1] if len(lines[i + 3].split("Response:")) > 1 else "" + backend = ( + next( + ( + api + for api in self.current_state.knowledge_base.get("apis", []) + if api["endpoint"] == endpoint.strip() + ), + {}, + ) + .get("locations", {}) + .get("backend", None) + ) + apis.append( + { + "description": description.strip(), + "endpoint": endpoint.strip(), + "request": request.strip(), + "response": response.strip(), + "locations": { + "frontend": { + "path": file.path, + "line": i - 1, + }, + "backend": backend, + }, + "status": "implemented" if backend is not None else "mocked", + } + ) + return apis + + async def update_apis(self, files_with_implemented_apis: list[dict] = []): + """ + Update the list of APIs. + + """ + apis = await self.get_apis() + for file in files_with_implemented_apis: + for endpoint in file["related_api_endpoints"]: + api = next((api for api in apis if (endpoint in api["endpoint"])), None) + if api is not None: + api["status"] = "implemented" + api["locations"]["backend"] = { + "path": file["path"], + "line": file["line"], + } + self.next_state.knowledge_base["apis"] = apis + self.next_state.flag_knowledge_base_as_modified() + await self.ui.knowledge_base_update(self.next_state.knowledge_base) + + @staticmethod + def get_input_required(content: str, file_path: str) -> list[int]: + """ + Get the list of lines containing INPUT_REQUIRED keyword. + + :param content: The file content to search. + :param file_path: The file path. + :return: Indices of lines with INPUT_REQUIRED keyword, starting from 1. + """ + lines = [] + + if ".env" not in file_path: + return lines + + for i, line in enumerate(content.splitlines(), start=1): + if "INPUT_REQUIRED" in line: + lines.append(i) + + return lines + + +__all__ = ["StateManager"] diff --git a/core/telemetry/__init__.py b/core/telemetry/__init__.py new file mode 100644 index 00000000..4f079ba8 --- /dev/null +++ b/core/telemetry/__init__.py @@ -0,0 +1,400 @@ +import sys +import time +import traceback +from copy import deepcopy +from os import getenv +from pathlib import Path +from typing import Any + +import httpx + +from core.config import get_config +from core.config.user_settings import settings +from core.config.version import get_version +from core.log import get_logger + +log = get_logger(__name__) + +LARGE_REQUEST_THRESHOLD = 50000 # tokens +SLOW_REQUEST_THRESHOLD = 300 # seconds + + +class Telemetry: + """ + Pythagora telemetry data collection. + + This class is a singleton, use the `telemetry` global variable to access it: + + >>> from core.telemetry import telemetry + + To record start of application creation process: + + >>> telemetry.start() + + To record data or increase counters: + + >>> telemetry.set("model", "gpt-4") + >>> telemetry.inc("num_llm_requests", 5) + + To stop recording and send the data: + + >>> telemetry.stop() + >>> await telemetry.send() + + Note: all methods are no-ops if telemetry is not enabled. + """ + + MAX_CRASH_FRAMES = 3 + + def __init__(self): + self.enabled = False + self.telemetry_id = None + self.endpoint = None + self.clear_data() + + if settings.telemetry is not None: + self.enabled = settings.telemetry.enabled + self.telemetry_id = settings.telemetry.id + self.endpoint = settings.telemetry.endpoint + + if self.enabled: + log.debug(f"Telemetry enabled (id={self.telemetry_id}), configure or disable it in {settings.config_path}") + + def clear_data(self): + """ + Reset all telemetry data to default values. + """ + config = get_config() + + self.data = { + # System platform + "platform": sys.platform, + # Python version used + "python_version": sys.version, + # Core version + "pilot_version": get_version(), + # Pythagora VSCode Extension version + "extension_version": None, + # Is extension used + "is_extension": False, + # The default LLM provider and model + "provider": config.agent["default"].provider.value, + "model": config.agent["default"].model, + # Initial prompt + "initial_prompt": None, + # Updated prompt + "updated_prompt": None, + # App complexity + "is_complex_app": None, + # Optional templates used for the project + "templates": None, + # Optional, example project selected by the user + "example_project": None, + # Optional user contact email + "user_contact": None, + # Unique project ID (app_id) + "app_id": None, + # Project architecture + "architecture": None, + # Documentation sets used for a given task + "docsets_used": [], + # Number of documentation snippets stored for a given task + "doc_snippets_stored": 0, + } + if sys.platform == "linux": + try: + import distro + + self.data["linux_distro"] = distro.name(pretty=True) + except Exception as err: + log.debug(f"Error getting Linux distribution info: {err}", exc_info=True) + self.clear_counters() + + def clear_counters(self): + """ + Reset telemetry counters while keeping the base data. + """ + self.data.update( + { + # Number of LLM requests made + "num_llm_requests": 0, + # Number of LLM requests that resulted in an error + "num_llm_errors": 0, + # Number of tokens used for LLM requests + "num_llm_tokens": 0, + # Number of development steps + "num_steps": 0, + # Number of commands run during development + "num_commands": 0, + # Number of times a human input was required during development + "num_inputs": 0, + # Number of files in the project + "num_files": 0, + # Total number of lines in the project + "num_lines": 0, + # Number of tasks started during development + "num_tasks": 0, + # Number of seconds elapsed during development + "elapsed_time": 0, + # Total number of lines created by Pythagora + "created_lines": 0, + # End result of development: + # - success:initial-project + # - success:feature + # - success:exit + # - failure + # - failure:api-error + # - interrupt + "end_result": None, + # Whether the project is continuation of a previous session + "is_continuation": False, + # Optional user feedback + "user_feedback": None, + # If Core crashes, record diagnostics + "crash_diagnostics": None, + # Statistics for large requests + "large_requests": None, + # Statistics for slow requests + "slow_requests": None, + } + ) + self.start_time = None + self.end_time = None + self.large_requests = [] + self.slow_requests = [] + + def set(self, name: str, value: Any): + """ + Set a telemetry data field to a value. + + :param name: name of the telemetry data field + :param value: value to set the field to + + Note: only known data fields may be set, see `Telemetry.clear_data()` for a list. + """ + if name not in self.data: + log.error(f"Telemetry.record(): ignoring unknown telemetry data field: {name}") + return + + self.data[name] = value + + def inc(self, name: str, value: int = 1): + """ + Increase a telemetry data field by a value. + + :param name: name of the telemetry data field + :param value: value to increase the field by (default: 1) + + Note: only known data fields may be increased, see `Telemetry.clear_data()` for a list. + """ + if name not in self.data: + log.error(f"Telemetry.increase(): ignoring unknown telemetry data field: {name}") + return + + self.data[name] += value + + def start(self): + """ + Record start of application creation process. + """ + self.start_time = time.time() + self.end_time = None + + def stop(self): + """ + Record end of application creation process. + """ + if self.start_time is None: + log.error("Telemetry.stop(): cannot stop telemetry, it was never started") + return + + self.end_time = time.time() + self.data["elapsed_time"] = int(self.end_time - self.start_time) + + def record_crash( + self, + exception: Exception, + end_result: str = "failure", + ) -> str: + """ + Record crash diagnostics. + + The formatted stack trace only contains frames from the `core` package + of gpt-pilot. + + :param exception: exception that caused the crash + :param end_result: end result of the application (default: "failure") + :return: formatted stack trace of the exception + + Records the following crash diagnostics data: + * formatted stack trace + * exception (class name and message) + * file:line for the last (innermost) 3 frames of the stack trace, only counting + the frames from the `core` package. + """ + self.set("end_result", end_result) + + root_dir = Path(__file__).parent.parent.parent + exception_class_name = exception.__class__.__name__ + exception_message = str(exception) + + frames = [] + info = [] + + for frame in traceback.extract_tb(exception.__traceback__): + try: + file_path = Path(frame.filename).absolute().relative_to(root_dir).as_posix() + except ValueError: + # outside of root_dir + continue + + if not file_path.startswith("core/"): + continue + + frames.append( + { + "file": file_path, + "line": frame.lineno, + "name": frame.name, + "code": frame.line, + } + ) + info.append(f"File `{file_path}`, line {frame.lineno}, in {frame.name}\n {frame.line}") + + frames.reverse() + stack_trace = "\n".join(info) + f"\n{exception.__class__.__name__}: {str(exception)}" + + self.data["crash_diagnostics"] = { + "stack_trace": stack_trace, + "exception_class": exception_class_name, + "exception_message": exception_message, + "frames": frames[: self.MAX_CRASH_FRAMES], + } + return stack_trace + + def record_llm_request( + self, + tokens: int, + elapsed_time: int, + is_error: bool, + ): + """ + Record an LLM request. + + :param tokens: number of tokens in the request + :param elapsed_time: time elapsed for the request + :param is_error: whether the request resulted in an error + """ + self.inc("num_llm_requests") + + if is_error: + self.inc("num_llm_errors") + else: + self.inc("num_llm_tokens", tokens) + + if tokens > LARGE_REQUEST_THRESHOLD: + self.large_requests.append(tokens) + if elapsed_time > SLOW_REQUEST_THRESHOLD: + self.slow_requests.append(elapsed_time) + + def calculate_statistics(self): + """ + Calculate statistics for large and slow requests. + """ + n_large = len(self.large_requests) + n_slow = len(self.slow_requests) + + self.data["large_requests"] = { + "num_requests": n_large, + "min_tokens": min(self.large_requests) if n_large > 0 else None, + "max_tokens": max(self.large_requests) if n_large > 0 else None, + "avg_tokens": sum(self.large_requests) // n_large if n_large > 0 else None, + "median_tokens": sorted(self.large_requests)[n_large // 2] if n_large > 0 else None, + } + self.data["slow_requests"] = { + "num_requests": n_slow, + "min_time": min(self.slow_requests) if n_slow > 0 else None, + "max_time": max(self.slow_requests) if n_slow > 0 else None, + "avg_time": sum(self.slow_requests) // n_slow if n_slow > 0 else None, + "median_time": sorted(self.slow_requests)[n_slow // 2] if n_slow > 0 else None, + } + + async def send(self, event: str = "pythagora-core-telemetry"): + """ + Send telemetry data to the phone-home endpoint. + + Note: this method clears all telemetry data after sending it. + """ + if not self.enabled or getenv("DISABLE_TELEMETRY"): + log.debug("Telemetry.send(): telemetry is disabled, not sending data") + return + + if self.endpoint is None: + log.error("Telemetry.send(): cannot send telemetry, no endpoint configured") + return + + if self.start_time is not None and self.end_time is None: + self.stop() + + self.calculate_statistics() + payload = { + "pathId": self.telemetry_id, + "event": event, + "data": self.data, + } + + log.debug(f"Telemetry.send(): sending telemetry data to {self.endpoint}") + try: + async with httpx.AsyncClient() as client: + response = await client.post(self.endpoint, json=payload) + response.raise_for_status() + self.clear_counters() + self.set("is_continuation", True) + except httpx.RequestError as e: + log.error(f"Telemetry.send(): failed to send telemetry data: {e}", exc_info=True) + + def get_project_stats(self) -> dict: + return { + "num_lines": self.data["num_lines"], + "num_files": self.data["num_files"], + "num_tokens": self.data["num_llm_tokens"], + } + + async def trace_code_event(self, name: str, data: dict): + """ + Record a code event to trace potential logic bugs. + + :param name: name of the event + :param data: data to send with the event + """ + if not self.enabled or getenv("DISABLE_TELEMETRY"): + return + + data = deepcopy(data) + for item in ["app_id", "user_contact", "platform", "pilot_version", "model"]: + data[item] = self.data[item] + + payload = { + "pathId": self.telemetry_id, + "event": f"trace-{name}", + "data": data, + } + + log.debug(f"Sending trace event {name} to {self.endpoint}: {repr(payload)}") + + try: + async with httpx.AsyncClient() as client: + await client.post(self.endpoint, json=payload) + except httpx.RequestError as e: + log.error(f"Failed to send trace event {name}: {e}", exc_info=True) + + async def trace_loop(self, name: str, task_with_loop: dict): + payload = deepcopy(self.data) + payload["task_with_loop"] = task_with_loop + await self.trace_code_event(name, payload) + + +telemetry = Telemetry() + + +__all__ = ["telemetry"] diff --git a/core/templates/__init__.py b/core/templates/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/core/templates/base.py b/core/templates/base.py new file mode 100644 index 00000000..6f2e0c49 --- /dev/null +++ b/core/templates/base.py @@ -0,0 +1,144 @@ +from json import loads +from os.path import dirname, join +from typing import TYPE_CHECKING, Any, Optional, Type +from uuid import uuid4 + +from pydantic import BaseModel + +from core.log import get_logger +from core.templates.render import Renderer + +if TYPE_CHECKING: + from core.proc.process_manager import ProcessManager + from core.state.state_manager import StateManager + +log = get_logger(__name__) + + +class NoOptions(BaseModel): + """ + Options class for templates that do not require any options. + """ + + class Config: + extra = "allow" + + pass + + +class BaseProjectTemplate: + """ + Base project template, providing a common interface for all project templates. + """ + + name: str + path: str + description: str + options_class: Type[BaseModel] + options_description: str + file_descriptions: dict + + def __init__( + self, + options: BaseModel, + state_manager: "StateManager", + process_manager: "ProcessManager", + ): + """ + Create a new project template. + + :param options: The options to use for the template. + :param state_manager: The state manager instance to save files to. + :param process_manager: ProcessManager instance to run the install commands. + """ + if isinstance(options, dict): + options = self.options_class(**options) + + self.options = options + self.state_manager = state_manager + self.process_manager = process_manager + + self.file_renderer = Renderer(join(dirname(__file__), "tree")) + self.info_renderer = Renderer(join(dirname(__file__), "info")) + + def filter(self, path: str) -> Optional[str]: + """ + Filter a file path to be included in the rendered template. + + The method is called for every file in the template tree before rendering. + If the method returns None or an empty string, the file will be skipped. + Otherwise, the file will be rendered and stored under the file name + matching the provided filename. + + By default (base template), this function returns the path as-is. + + :param path: The file path to include or exclude. + :return: The path to use, or None if the file should be skipped. + """ + return path + + async def apply(self) -> Optional[str]: + """ + Apply a project template to a new project. + + :param template_name: The name of the template to apply. + :param state_manager: The state manager instance to save files to. + :param process_manager: The process manager instance to run install hooks with. + :return: A summary of the applied template, or None if no template was applied. + """ + state = self.state_manager.current_state + project_name = state.branch.project.name + project_folder = state.branch.project.folder_name + project_description = state.specification.description + + log.info(f"Applying project template {self.name} with options: {self.options_dict}") + + files = self.file_renderer.render_tree( + self.path, + { + "project_name": project_name, + "project_folder": project_folder, + "project_description": project_description, + "random_secret": uuid4().hex, + "options": self.options_dict, + }, + self.state_manager.file_system.root, + self.filter, + ) + + for file_name, file_content in files.items(): + desc = self.file_descriptions.get(file_name) + metadata = {"description": desc} if desc else None + await self.state_manager.save_file( + file_name, + file_content, + metadata=metadata, + from_template=True, + ) + + try: + await self.install_hook() + except Exception as err: + log.error( + f"Error running install hook for project template '{self.name}': {err}", + exc_info=True, + ) + + return self.info_renderer.render_template( + join(self.path, "summary.tpl"), + { + "description": self.description, + "options": self.options, + }, + ) + + async def install_hook(self): + """ + Command to run to complete the project scaffolding setup. + """ + raise NotImplementedError() + + @property + def options_dict(self) -> dict[str, Any]: + """Template options as a Python dictionary.""" + return loads(self.options.model_dump_json()) diff --git a/core/templates/example_project.py b/core/templates/example_project.py new file mode 100644 index 00000000..2ef8a476 --- /dev/null +++ b/core/templates/example_project.py @@ -0,0 +1,83 @@ +from core.db.models import Complexity + +EXAMPLE_PROJECT_DESCRIPTION = """ +The application is a simple ToDo app built using React. Its primary function is to allow users to manage a list of tasks (todos). Each task has a description and a state (open or completed, with the default state being open). The application is frontend-only, with no user sign-up or authentication process. The goal is to provide a straightforward and user-friendly interface for task management. + +Features: +1. Display of Todos: A list that displays all todo items. Each item shows its description and a checkbox to indicate its state (open or completed). +2. Add New Todo: A button to add a new todo item. Clicking this button will prompt the user to enter a description for the new todo. +3. Toggle State: Each todo item includes a checkbox. Checking/unchecking this box toggles the todo's state between open and completed. +4. Local Storage: The application will use the browser's local storage to persist todos between sessions, ensuring that users do not lose their data upon reloading the application. + +Functional Specification: +- Upon loading the application, it fetches existing todos from the local storage and displays them in a list. +- Each todo item in the list displays a checkbox and a description. The checkbox reflects the todo's current state (checked for completed, unchecked for open). +- When the user checks or unchecks a checkbox, the application updates the state of the corresponding todo item and saves the updated list to local storage. +- Clicking the "Add New Todo" button prompts the user to enter a description for the new todo. Upon confirmation, the application adds the new todo (with the default state of open) to the list and updates local storage. +- The application does not support deleting or editing todo items to keep the interface and interactions simple. +- Todos persist between sessions using the browser's local storage. The application saves any changes to the todo list (additions or state changes) in local storage and retrieves this data when the application is reloaded. + +Technical Specification: +- Platform/Technologies: The application is a web application developed using React on frontend and Express on the backend, using SQLite as the database. +- Styling: Use Bootstrap 5 for a simple and functional interface. Load Boostrap from the CDN (don't install it locally): + - https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/css/bootstrap.min.css + - https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/js/bootstrap.bundle.min.js +- State Management: Directly in the React component + - make sure to initialize the state from the local storage as default (... = useState(JSON.parse(localStorage.getItem('todos')) || []) to avoid race conditions +- Data Persistence: Using the SQLite database on the backend via a REST API. +""" + +EXAMPLE_PROJECT_ARCHITECTURE = { + "architecture": ( + "The application is a client-side React web application that uses local storage for data persistence. " + "It consists of a single page with components for listing todos, adding new todos, and toggling their completion status. " + "State management is handled directly within React components, leveraging useState and useEffect hooks for state manipulation and side effects, respectively. " + "Bootstrap 5 is used for styling to provide a responsive and accessible UI." + ), + "system_dependencies": [ + { + "name": "Node.js", + "description": "JavaScript runtime needed to run the React development tools and build the project.", + "test": "node --version", + "required_locally": True, + } + ], + "package_dependencies": [ + {"name": "react", "description": "A JavaScript library for building user interfaces."}, + {"name": "react-dom", "description": "Serves as the entry point to the DOM and server renderers for React."}, + {"name": "bootstrap", "description": "Frontend framework for developing responsive and mobile-first websites."}, + ], + "templates": { + "javascript_react": {}, + }, +} + +EXAMPLE_PROJECT_PLAN = [ + { + "description": ( + "Create a new component TodoList: This component will display the list of todo items. " + "Use localStorage directly to access the current state of todos and map over them, rendering each todo item as a list item. " + "Each item should display the todo's description and a checkbox that reflects the todo's state (checked for completed, unchecked for open). " + "When the checkbox is clicked, dispatch an action to toggle the state of the todo. " + "Also create AddTodo: This component will include a button that, when clicked, displays a prompt asking the user for a description of the new todo. " + "Upon confirmation, dispatch an action to add the new todo to the state with a default state of open. " + "Ensure the component also updates the local storage with the new list of todos. " + "Finally, use TodoList and AddTodo components in App component to implement the required functionality. " + "Integrate Boostrap 5 for styling - add CSS/JS to index.html, style App.jsx and other files as appropriate." + ), + "status": "todo", + "sub_epic_id": 1, + } +] + + +EXAMPLE_PROJECTS = { + "example-project": { + "description": EXAMPLE_PROJECT_DESCRIPTION, + "architecture": EXAMPLE_PROJECT_ARCHITECTURE, + "complexity": Complexity.SIMPLE, + "plan": EXAMPLE_PROJECT_PLAN, + } +} + +DEFAULT_EXAMPLE_PROJECT = "example-project" diff --git a/core/templates/info/javascript_react/summary.tpl b/core/templates/info/javascript_react/summary.tpl new file mode 100644 index 00000000..a651cd97 --- /dev/null +++ b/core/templates/info/javascript_react/summary.tpl @@ -0,0 +1,9 @@ +Here's what's already been implemented: + +* React web app using Vite devserver/bundler +* Initial setup with Vite for fast development +* Basic project structure for React development +* Development server setup for hot reloading +* Minimal configuration to get started with React +* Frontend-only, compatible with any backend stack + diff --git a/core/templates/info/node_express_mongoose/summary.tpl b/core/templates/info/node_express_mongoose/summary.tpl new file mode 100644 index 00000000..7e2839c3 --- /dev/null +++ b/core/templates/info/node_express_mongoose/summary.tpl @@ -0,0 +1,10 @@ +Here's what's already been implemented: + +* Node + Express + MongoDB web app with session-based authentication, EJS views and Bootstrap 5 +* initial Node + Express setup +* User model in Mongoose ORM with username and password fields, ensuring username is unique and hashing passwords with bcrypt prior to saving to the database +* session-based authentication using username + password (hashed using bcrypt) in routes/authRoutes.js, using express-session +* authentication middleware to protect routes that require login +* EJS view engine, html head, header and footer EJS partials, with included Boostrap 5.x CSS and JS +* routes and EJS views for login, register, and home (main) page +* config loading from environment using dotenv with a placeholder .env.example file: you will need to create a .env file with your own values diff --git a/core/templates/info/react_express/summary.tpl b/core/templates/info/react_express/summary.tpl new file mode 100644 index 00000000..efb8bbc4 --- /dev/null +++ b/core/templates/info/react_express/summary.tpl @@ -0,0 +1,39 @@ +Here's what's already been implemented: + +* {{ description }} +* Frontend: + * ReactJS based frontend in `ui/` folder using Vite devserver + * Integrated shadcn-ui component library with Tailwind CSS framework + * Client-side routing using `react-router-dom` with page components defined in `ui/pages/` and other components in `ui/components` + * Implememented pages: + * Home - home (index) page (`/`) + {% if options.auth %} + * Login - login page (`/login/`) - on login, stores the auth token to `token` variable in local storage + * Register - register page (`/register/`) + {% endif %} +* Backend: + * Express-based server implementing REST API endpoints in `api/` + {% if options.db_type == "sql" %} + * Relational (SQL) database support with Prisma ORM using SQLite as the database + {% elif options.db_type == "nosql" %} + * MongoDB database support with Mongoose + {% endif %} + {% if options.email %} + * Email sending support using Nodemailer + {% endif %} + {% if options.auth %} + * Token-based authentication (using opaque bearer tokens) + * User authentication (email + password): + * login/register API endpoints in `/api/routes/authRoutes.js` + * authorization middleware in `/api/middlewares/authMiddleware.js` + * user management logic in `/api/services/userService.js` + {% endif %} +* Development server: + * Vite devserver for frontend (`npm run dev:ui` to start the Vite dev server) + * Nodemon for backend (`npm run dev:api` to start Node.js server with Nodemon) + * Concurrently to run both servers together with a single command (`npm run dev`) - the preferred way to start the server in development +* Notes: + {% if options.db_type == "sql" %} + * Whenever a database model is changed or added in `schema.prisma`, remember to run `npx prisma format && npx prisma generate` to update the Prisma client + * For model relationships, remember to always also add the reverse relationship in `schema.prisma` at the same time, otherwise the database migration will fail + {% endif %} diff --git a/core/templates/info/vite_react/summary.tpl b/core/templates/info/vite_react/summary.tpl new file mode 100644 index 00000000..41f8b752 --- /dev/null +++ b/core/templates/info/vite_react/summary.tpl @@ -0,0 +1,91 @@ +IMPORTANT: +This app has 2 parts: + +** #1 Frontend ** + * ReactJS based frontend in `client/` folder using Vite devserver + * Integrated shadcn-ui component library with Tailwind CSS framework + * Client-side routing using `react-router-dom` with page components defined in `client/src/pages/` and other components in `client/src/components` + * It is running on port 5173 and this port should be used for user testing when possible + * All requests to the backend need to go to an endpoint that starts with `/api/` (e.g. `/api/companies`) + * Implememented pages: + * Home - home (index) page (`/`){% if options.auth %} + * Login - login page (`/login/`) - on login, stores the auth tokens to `accessToken` and `refreshToken` variables in local storage + * Register - register page (`/register/`) - on register, store **ONLY** the `accessToken` variable in local storage{% endif %} + +** #2 Backend ** + * Express-based server implementing REST API endpoints in `api/` + * Has codebase inside "server/" folder + * Backend is running on port 3000 + * MongoDB database support with Mongoose{% if options.auth %} + * Token-based authentication (using bearer access and refresh tokens) + * User authentication (email + password): + * login/register API endpoints in `/server/routes/auth.js` + * authorization middleware in `/server/routes/middleware/auth.js` + * user management logic in `/server/routes/services/user.js` + * User authentication is implemented and doesn't require any additional work{% endif %} + + +Concurrently is used to run both client and server together with a single command (`npm run start`). + +** IMPORTANT - Mocking data on the frontend ** +All API requests from the frontend to the backend must be defined in files inside the api folder (you must never make requests directly from the components) and the data must be mocked during the frontend implementation. Make sure to always add an API request whenever something needs to be sent or fetched from the backend. +When you add mock data, make sure to mock the data in files in the `client/src/api` folder and above each mocked API request, add a structure that is expected from the API with fields `Description`, `Endpoint`, `Request`, and `Response`. You **MUST NOT** add mock data anywhere else in the frontend codebase. +Mocking example: + +The base client/src/api/api.ts is already created so here are 2 examples for how to write functions to get data from the backend with the mock data: +β€”EXAMPLE_1 (file `client/src/api/companies.ts`) β€” +import api from './api'; + +// Description: Get a list of Companies +// Endpoint: GET /api/companies +// Request: {} +// Response: { companies: Array<{ domain: string, name: string, lastContact: string }> } +export const getCompanies = () => { + // Mocking the response + return new Promise((resolve) => { + setTimeout(() => { + resolve({ + companies: [ + {domain: 'google.com', name: 'Google', lastContact: '2021-08-01'}, + {domain: 'facebook.com', name: 'Facebook', lastContact: '2021-08-02'}, + {domain: 'microsoft.com', name: 'Microsoft', lastContact: '2021-08-03'}, + ], + }); + }, 500); + }); + // Uncomment the below lines to make an actual API call + // try { + // return await api.get('/api/companies', data); + // } catch (error) { + // throw new Error(error?.response?.data?.error || error.message); + // } +} +β€”END_OF_EXAMPLE_1β€” + +β€”EXAMPLE_2 (file `client/src/api/work.ts`) β€” +import api from './api'; + +// Description: Add a new Work +// Endpoint: POST /api/work +// Request: { work: string, driveLink: string } +// Response: { success: boolean, message: string } +export const addWork = (data: { work: string; driveLink: string }) => { + // Mocking the response + return new Promise((resolve) => { + setTimeout(() => { + resolve({success: true, message: 'Work added successfully'}); + }, 500); + }); + // Uncomment the below lines to make an actual API call + // try { + // return await api.post('/api/work/add', data); + // } catch (error) { + // throw new Error(error?.response?.data?.error || error.message); + // } +} +β€”END_OF_EXAMPLE_2β€” + +Whenever you add an API request from the frontend, make sure to wrap the request in try/catch block and in the catch block, return `throw new Error(error?.response?.data?.message || error.message);` - in the place where the API request function is being called, show a toast message with an error. + +**IMPORTANT** +Mongodb is being used as a database so whenever you need to take an `id` of an object on frontend, make sure to take `_id`. For example, if you have a company object, whenever you want to set an id for an element, you should get `company._id` instead of `company.id`. diff --git a/core/templates/javascript_react.py b/core/templates/javascript_react.py new file mode 100644 index 00000000..06049c19 --- /dev/null +++ b/core/templates/javascript_react.py @@ -0,0 +1,35 @@ +from .base import BaseProjectTemplate, NoOptions + + +class JavascriptReactProjectTemplate(BaseProjectTemplate): + stack = "frontend" + name = "javascript_react" + path = "javascript_react" + description = "React web app using Vite devserver/bundler" + file_descriptions = { + "vite.config.js": "Configuration file for Vite, a fast developer-friendly Javascript bundler/devserver.", + "index.html": "Main entry point for the project. It includes a basic HTML structure with a root div element and a script tag importing a JavaScript file named main.jsx using the module type. References: src/main.jsx", + ".eslintrc.cjs": "Configuration file for ESLint, a static code analysis tool for identifying problematic patterns found in JavaScript code. It defines rules for linting JavaScript code with a focus on React applications.", + ".gitignore": "Specifies patterns to exclude files and directories from being tracked by Git version control system. It is used to prevent certain files from being committed to the repository.", + "package.json": "Standard Nodejs package metadata file, specifies dependencies and start scripts. It also specifies that the project is a module.", + "public/.gitkeep": "Empty file", + "src/app.css": "Contains styling rules for the root element of the application, setting a maximum width, centering it on the page, adding padding, and aligning text to the center.", + "src/index.css": "Defines styling rules for the root element, body, and h1 elements of a web page.", + "src/App.jsx": "Defines a functional component that serves as the root component in the project. The component is exported as the default export. References: src/app.css", + "src/main.jsx": "Main entry point for a React application. It imports necessary modules, renders the main component 'App' inside a 'React.StrictMode' component, and mounts it to the root element in the HTML document. References: App.jsx, index.css", + "src/assets/.gitkeep": "Empty file", + } + summary = "\n".join( + [ + "* Initial setup with Vite for fast development", + "* Basic project structure for React development", + "* Development server setup for hot reloading", + "* Minimal configuration to get started with React", + "* Frontend-only, compatible with any backend stack", + ] + ) + options_class = NoOptions + options_description = "" + + async def install_hook(self): + await self.process_manager.run_command("npm install") diff --git a/core/templates/node_express_mongoose.py b/core/templates/node_express_mongoose.py new file mode 100644 index 00000000..50d83840 --- /dev/null +++ b/core/templates/node_express_mongoose.py @@ -0,0 +1,41 @@ +from .base import BaseProjectTemplate, NoOptions + + +class NodeExpressMongooseProjectTemplate(BaseProjectTemplate): + stack = "backend" + name = "node_express_mongoose" + path = "node_express_mongoose" + description = "Node + Express + MongoDB web app with session-based authentication, EJS views and Bootstrap 5" + file_descriptions = { + ".env.example": "The .env.example file serves as a template for setting up environment variables used in the application. It provides placeholders for values such as the port number, MongoDB database URL, and session secret string.", + ".env": "This file is a configuration file in the form of a .env file. It contains environment variables used by the application, such as the port to listen on, the MongoDB database URL, and the session secret string.", + "server.js": "This `server.js` file sets up an Express server with MongoDB database connection, session management using connect-mongo, templating engine EJS, static file serving, authentication routes, error handling, and request logging. [References: dotenv, mongoose, express, express-session, connect-mongo, ./routes/authRoutes]", + "package.json": "This `package.json` file is used to define the metadata and dependencies for a Node.js project named 'tt0'. It specifies the project name, version, main entry point file, scripts for starting and testing the project, dependencies required by the project, and other metadata like author and license. [References: server.js]", + "views/login.ejs": "This file represents the login page of a web application using EJS (Embedded JavaScript) templating. It includes partials for the head, header, and footer sections, and contains a form for users to input their username and password to log in. [References: partials/_head.ejs, partials/_header.ejs, partials/_footer.ejs]", + "views/register.ejs": "The 'views/register.ejs' file contains the HTML markup for a registration form. It includes fields for username and password, along with a button to submit the form and a link to redirect to the login page if the user already has an account. [References: partials/_head.ejs, partials/_header.ejs, partials/_footer.ejs]", + "views/index.ejs": "This file represents the main view for a web application. It includes partials for the head, header, and footer sections, and contains a simple HTML structure with a main container displaying a heading. [References: partials/_head.ejs, partials/_header.ejs, partials/_footer.ejs, js/main.js]", + "views/partials/_header.ejs": "This file represents a partial view for the header section of a web page. It includes a navigation bar with a brand logo, toggle button, and links for Home, Login, and Logout based on the user's session status.", + "views/partials/_head.ejs": "This file represents the partial for the head section of an HTML document. It includes meta tags, a title tag, and links to external CSS files (Bootstrap and a custom stylesheet).", + "views/partials/_footer.ejs": "This file defines the footer section of a web page using EJS (Embedded JavaScript) templating. It includes a copyright notice and a link to the Bootstrap JavaScript library. [References: https://cdn.jsdelivr.net/npm/bootstrap@5.3.2/dist/js/bootstrap.min.js]", + "routes/authRoutes.js": "This file defines routes for user authentication including registration, login, and logout. It interacts with a User model to handle user data and uses bcrypt for password hashing and comparison. [References: models/User.js]", + "routes/middleware/authMiddleware.js": "This file defines a middleware function called isAuthenticated, which checks if a user is authenticated based on the presence of a userId in the session object. If authenticated, it allows the request to proceed to the next middleware or route handler; otherwise, it returns a 401 status response indicating the user is not authenticated.", + "models/User.js": "This file defines a Mongoose model for a user with fields for username and password. It includes a pre-save hook to hash the user's password before saving it to the database using bcrypt. [References: mongoose, bcrypt]", + "public/js/main.js": "The main.js file is a placeholder for future JavaScript code. It currently does not contain any specific functionality.", + "public/css/style.css": "This file is a placeholder for custom styles. It does not contain any specific styles but is intended for adding custom CSS styles.", + } + summary = "\n".join( + [ + "* initial Node + Express setup", + "* User model in Mongoose ORM with username and password fields, ensuring username is unique and hashing passwords with bcrypt prior to saving to the database", + "* session-based authentication using username + password (hashed using bcrypt) in routes/authRoutes.js, using express-session", + "* authentication middleware to protect routes that require login", + "* EJS view engine, html head, header and footer EJS partials, with included Boostrap 5.x CSS and JS", + "* routes and EJS views for login, register, and home (main) page", + "* config loading from environment using dotenv with a placeholder .env.example file: you will need to create a .env file with your own values", + ] + ) + options_class = NoOptions + options_description = "" + + async def install_hook(self): + await self.process_manager.run_command("npm install") diff --git a/core/templates/react_express.py b/core/templates/react_express.py new file mode 100644 index 00000000..ee51f479 --- /dev/null +++ b/core/templates/react_express.py @@ -0,0 +1,122 @@ +from enum import Enum +from typing import Optional + +from pydantic import BaseModel, Field + +from core.log import get_logger + +from .base import BaseProjectTemplate + +log = get_logger(__name__) + + +class DatabaseType(str, Enum): + SQL = "sql" + NOSQL = "nosql" + NONE = "none" + + +class TemplateOptions(BaseModel): + db_type: DatabaseType = Field( + DatabaseType.NONE, + description="Type of database to use in the project: relational/SQL (eg SQLite or Postgres), nosql (eg Mongo or Redis) or no database at all", + ) + auth: bool = Field( + description="Whether the app supports users and email/password authentication", + ) + + +TEMPLATE_OPTIONS = """ +* Database Type (`db_type`): What type of database should the project use: SQL (relational database like SQLite or Postgres), NoSQL (MongoDB, Redis), or no database at all. +* Authentication (`auth`): Does the project support users registering and authenticating (using email/password). +""" + + +class ReactExpressProjectTemplate(BaseProjectTemplate): + stack = "fullstack" + name = "react_express" + path = "react_express" + description = "React frontend with Node/Express REST API backend" + file_descriptions = { + ".babelrc": "Configuration file used by Babel, a JavaScript transpiler, to define presets for transforming code. In this specific file, two presets are defined: 'env' with a target of 'node' set to 'current', and 'jest' for Jest testing framework.", + ".env": "Contains environment variables used to configure the application. It specifies the Node environment, log level, port to listen on, database provider and URL, as well as the session secret string.", + ".eslintrc.json": "Contains ESLint configuration settings for the project. It specifies the environment (browser, ES2021, Node.js, Jest), extends the ESLint recommended rules, sets parser options for ECMAScript version 12 and module source type, and defines a custom rule to flag unused variables except for 'req', 'res', and 'next' parameters.", + ".gitignore": "Specifies patterns to exclude certain files and directories from being tracked by Git version control. It helps in preventing unnecessary files from being committed to the repository.", + "README.md": "Main README for a time-tracking web app for freelancers. The app uses React for the frontend, Node/Express for the backend, Prisma ORM, and SQLite database. It also utilizes Bootstrap for UI styling. The app allows users to register with email and password, uses opaque bearer tokens for authentication, and provides features like time tracking, saving time entries, viewing recent entries, generating reports, and exporting time entries in CSV format. The README also includes instructions for installation, development, testing, production deployment, and Docker usage.", + "api/app.js": "Sets up an Express app for handling API routes and serving a pre-built frontend. It enables CORS, parses JSON and URL-encoded data, serves static files, and defines routes for authentication and API endpoints. Additionally, it serves the pre-built frontend from the '../dist' folder for all other routes.", + "api/middlewares/authMiddleware.js": "Implements middleware functions for authentication and user authorization. The 'authenticateWithToken' function checks the Authorization header in the request, extracts the token, and authenticates the user using the UserService. The 'requireUser' function ensures that a user is present in the request object before allowing access to subsequent routes.", + "api/middlewares/errorMiddleware.js": "Implements middleware functions for handling 404 and 500 errors in an Express API. The 'handle404' function is responsible for returning a 404 response when a requested resource is not found or an unsupported HTTP method is used. The 'handleError' function is used to handle errors that occur within route handlers by logging the error details and sending a 500 response.", + "api/models/init.js": "Initializes the database client for interacting with the database.", + "api/models/user.js": "Defines a Mongoose schema for a user in a database, including fields like email, password, token, name, creation date, last login date, and account status. It also includes methods for authenticating users with password or token, setting and regenerating passwords, and custom JSON transformation. The file exports a Mongoose model named 'User' based on the defined schema.", + "api/routes/authRoutes.js": "Defines routes related to user authentication using Express.js. It includes endpoints for user login, registration, logout, and password management. The file imports services, middlewares, and utilities required for handling authentication logic.", + "api/routes/index.js": "Defines the API routes using the Express framework. It creates an instance of the Express Router and exports it to be used in the main application. The routes defined in this file are expected to have a '/api/' prefix to differentiate them from UI/frontend routes.", + "api/services/userService.js": "Implements a UserService class that provides various methods for interacting with user data in the database. It includes functions for listing users, getting a user by ID or email, updating user information, deleting users, authenticating users with password or token, creating new users, setting user passwords, and regenerating user tokens. The class utilizes the 'crypto' library for generating random UUIDs and imports functions from 'password.js' for password hashing and validation.", + "api/utils/log.js": "Defines a logger utility using the 'pino' library for logging purposes. It sets the log level based on the environment variable 'LOG_LEVEL' or defaults to 'info' in production and 'debug' in other environments. It validates the provided log level against the available levels in 'pino' and throws an error if an invalid level is specified. The logger function creates a new logger instance with the specified name and log level.", + "api/utils/mail.js": "Implements a utility function to send emails using nodemailer. It reads configuration options from environment variables and creates a nodemailer transporter with the specified options. The main function exported from this file is used to send emails by passing the necessary parameters like 'from', 'to', 'subject', and 'text'.", + "api/utils/password.js": "Implements functions related to password hashing and validation using the bcrypt algorithm. It provides functions to generate a password hash, validate a password against a hash, and check the format of a hash.", + "index.html": "The main entry point for the web application front-end. It defines the basic structure of an HTML document with a title and a root div element where the application content will be rendered. Additionally, it includes a script tag that imports the main.jsx file as a module, indicating that this file contains JavaScript code to be executed in a modular fashion.", + "package.json": "Configuration file used for both Node.js/Express backend and React/Vite frontend define metadata about the project such as name, version, description, dependencies, devDependencies, scripts, etc. It also specifies the entry point of the application through the 'main' field.", + "prisma/schema.prisma": "Defines the Prisma ORM schema for the project. It specifies the data source configuration, generator settings, and a 'User' model with various fields like id, email, password, token, name, createdAt, lastLoginAt, and isActive. It also includes index definitions for 'email' and 'token' fields.", + "public/.gitkeep": "(empty file)", + "server.js": "The main entry point for the backend. It sets up an HTTP server using Node.js's 'http' module, loads environment variables using 'dotenv', imports the main application logic from 'app.js', and initializes a logger from 'log.js'. It also handles uncaught exceptions and unhandled rejections, logging errors and closing the server accordingly. The main function starts the server on a specified port, defaulting to 3000 if not provided in the environment variables.", + "ui/assets/.gitkeep": "(empty file)", + "ui/index.css": "Defines main styling rules for the user interface elements. It sets the root font properties, body layout, and heading styles.", + "ui/main.jsx": "Responsible for setting up the main UI components of the application using React and React Router. It imports necessary dependencies like React, ReactDOM, and react-router-dom. It also imports the main CSS file for styling. The file defines the main router configuration for the app, setting up the Home page to be displayed at the root path. Finally, it renders the main UI components using ReactDOM.createRoot.", + "ui/pages/Home.css": "Defines the styling for the home page of the UI. It sets the maximum width of the root element to 1280px, centers it horizontally on the page, adds padding around it, and aligns the text in the center.", + "ui/pages/Home.jsx": "Defines a functional component named 'Home' that gets displayed on the app home page (`/`). It imports styles from the 'Home.css' file.", + "vite.config.js": "The 'vite.config.js' file is used to configure the Vite build tool for a project. In this specific file, the configuration is defined using the 'defineConfig' function provided by Vite. It includes the 'react' plugin from '@vitejs/plugin-react' to enable React support in the project. The configuration sets up the plugins array with the 'react' plugin initialized.", + } + + summary = "\n".join( + [ + "* React-based frontend using Vite devserver", + "* Radix/Shadcn UI components with Tailwind CSS, and React Router", + "* Node.js/Express REST API backend", + "* Dotenv-based configuration", + "* Database integration - optional (MongoDB via Mongoose or SQL/relational via Prisma)", + "* User authentication (email+password) - optional", + ] + ) + + options_class = TemplateOptions + options_description = TEMPLATE_OPTIONS.strip() + + async def install_hook(self): + await self.process_manager.run_command("npm install") + if self.options.db_type == DatabaseType.SQL: + await self.process_manager.run_command("npx prisma generate") + await self.process_manager.run_command("npx prisma migrate dev --name initial") + + def filter(self, path: str) -> Optional[str]: + if not self.options.auth and path in [ + "api/middlewares/authMiddleware.js", + "api/models/user.js", + "api/routes/authRoutes.js", + "api/services/userService.js", + "api/utils/password.js", + "ui/pages/Login.jsx", + "ui/pages/Register.jsx", + ]: + log.debug(f"Skipping {path} as auth is disabled") + return None + + if self.options.db_type != DatabaseType.SQL.value and path in [ + "prisma/schema.prisma", + ]: + log.debug(f"Skipping {path} as ORM is not Prisma") + return None + + if self.options.db_type != DatabaseType.NOSQL.value and path in [ + "api/models/user.js", + ]: + log.debug(f"Skipping {path} as Orm is not Mongoose") + return None + + if self.options.db_type == DatabaseType.NONE.value and path in [ + "api/models/init.js", + ]: + log.debug(f"Skipping {path} as database integration is not enabled") + return None + + log.debug(f"Including project template file {path}") + return path diff --git a/core/templates/registry.py b/core/templates/registry.py new file mode 100644 index 00000000..492a3b9f --- /dev/null +++ b/core/templates/registry.py @@ -0,0 +1,28 @@ +from enum import Enum + +from core.log import get_logger + +# from .javascript_react import JavascriptReactProjectTemplate +# from .node_express_mongoose import NodeExpressMongooseProjectTemplate +from .vite_react import ViteReactProjectTemplate + +# from .react_express import ReactExpressProjectTemplate + +log = get_logger(__name__) + + +class ProjectTemplateEnum(str, Enum): + """Choices of available project templates.""" + + # JAVASCRIPT_REACT = JavascriptReactProjectTemplate.name + # NODE_EXPRESS_MONGOOSE = NodeExpressMongooseProjectTemplate.name + VITE_REACT = ViteReactProjectTemplate.name + # REACT_EXPRESS = ReactExpressProjectTemplate.name + + +PROJECT_TEMPLATES = { + # JavascriptReactProjectTemplate.name: JavascriptReactProjectTemplate, + # NodeExpressMongooseProjectTemplate.name: NodeExpressMongooseProjectTemplate, + ViteReactProjectTemplate.name: ViteReactProjectTemplate, + # ReactExpressProjectTemplate.name: ReactExpressProjectTemplate, +} diff --git a/core/templates/render.py b/core/templates/render.py new file mode 100644 index 00000000..dc3e4133 --- /dev/null +++ b/core/templates/render.py @@ -0,0 +1,135 @@ +from __future__ import annotations + +import os +from os import walk +from os.path import join, relpath +from pathlib import Path +from typing import Any, Callable + +from jinja2 import Environment, FileSystemLoader + + +def escape_string(str: str) -> str: + """ + Escape special characters in a string + + :param str: The string to escape + :return: The escaped string + """ + return str.replace("\\", "\\\\").replace('"', '\\"').replace("\n", "\\n") + + +class Renderer: + """ + Render a Jinja template + + Sets up Jinja renderer and renders one or more templates + using provided context. + + * `render_template` renders a single template + * `render_tree` renders all templates starting from a predefined + root folder (which must reside inside templates folder structure) + + Rendered template(s) are returned as strings. Nothing is written + to disk. + + Usage: + + >>> import Renderer from render + >>> r = Renderer('path/to/templates') + >>> output_string = r.render_template('template.html', {'key': 'value'}) + >>> output_tree = r.render_tree('tree/root', {'key': 'value'}) + """ + + def __init__(self, template_dir: str): + self.template_dir = template_dir + self.jinja_env = Environment( + loader=FileSystemLoader(template_dir), + autoescape=False, + lstrip_blocks=True, + trim_blocks=True, + keep_trailing_newline=True, + ) + # Add filters here + self.jinja_env.filters["escape_string"] = escape_string + + def render_template(self, template: str, context: Any) -> str: + """ + Render a single template to a string using provided context + + :param template: Name of the template file, relative to `template_dir`. + :param context: Context to render the template with. + :return: The resulting string. + """ + + # Jinja2 always uses /, even on Windows + template = template.replace("\\", "/") + + tpl_object = self.jinja_env.get_template(template) + return tpl_object.render(context) + + def render_tree(self, root: str, context: Any, full_root_dir: str, filter: Callable = None) -> dict[str, str]: + """ + Render a tree folder structure of templates using provided context + + :param root: Root of the tree (relative to `template_dir`). + :param context: Context to render the templates with. + :param full_root_dir: Full path to the root of the tree. + :param filter: If defined, will be called for each file to check if it + needs to be processed and determine output file path. + :return: A flat dictionary with path => content structure. + + Root must be inside the template_dir (and must be specified relative + to it), but need not be at the root of the template-dir. + + If supplied, `filter` must be a callable taking a single string + argument. It will be called for every file before processing it, with + the file name (relative to root of the tree) as the argument. If filter + returns a non-empty string, file will be rendered. If it returns None + or an empty string, file will be skipped. If `filter` is not defined, + all files are processed. + + In the returned structure, `file_name` is location of the file + relative to the tree root (unless changed by `filter`) and + `contents` is file contents rendered to a binary (utf8-encoded) string. + + Directories are implied by file paths, not represented by elements + in the returned dictionary. + """ + + retval = {} + + # Actual full path of the root of the tree we're rendering + full_root = join(self.template_dir, root) + + for path, subdirs, files in walk(full_root): + for file in files: + file_path = join(path, file) # actual full path of the template file + output_location = Path(file_path).relative_to(full_root).as_posix() # template relative to tree root + + # Skip .DS_Store files + if file == ".DS_Store": + continue + elif file.endswith( + (".png", ".jpg", ".jpeg", ".gif", ".ico", ".svg", ".woff", ".woff2", ".ttf", ".eot") + ): + with open(file_path, "rb") as f: + content = f.read() + final_path = join(full_root_dir, output_location) + os.makedirs(os.path.dirname(final_path), exist_ok=True) + with open(final_path, "wb") as out: + out.write(content) + continue + + tpl_location = relpath(file_path, self.template_dir) # template relative to template_dir + + if filter: + output_location = filter(output_location) + if not output_location: + continue + + contents = self.render_template(tpl_location, context) + if contents != "": + retval[output_location] = contents + + return retval diff --git a/core/templates/tree/add_raw_tags.py b/core/templates/tree/add_raw_tags.py new file mode 100644 index 00000000..574a6166 --- /dev/null +++ b/core/templates/tree/add_raw_tags.py @@ -0,0 +1,56 @@ +import os +import sys + + +def add_raw_tags_to_file(file_path): + """Add {% raw %} at the beginning and {% endraw %} at the end of the file, if not already present.""" + try: + # Open the file and read the contents + with open(file_path, "r", encoding="utf-8") as file: + content = file.read() + + # Check if the tags are already present + if content.startswith("{% raw %}") and content.endswith("{% endraw %}\n"): + print(f"Skipping file (tags already added): {file_path}") + return + + # Add {% raw %} at the beginning and {% endraw %} at the end + modified_content = f"{'{% raw %}'}\n{content}\n{'{% endraw %}'}" + + # Write the modified content back to the file + with open(file_path, "w", encoding="utf-8") as file: + file.write(modified_content) + + print(f"Processed file: {file_path}") + + except Exception as e: + print(f"Error processing {file_path}: {e}") + + +def process_directory(directory): + """Recursively process all files in the given directory.""" + for root, dirs, files in os.walk(directory): + for file in files: + # Construct the full file path + file_path = os.path.join(root, file) + + # Process the file + add_raw_tags_to_file(file_path) + + +if __name__ == "__main__": + # Check if the directory path argument is provided + if len(sys.argv) != 2: + print("Usage: python add_raw_tags.py ") + sys.exit(1) + + # Get the directory path from the command line argument + directory_path = sys.argv[1] + + # Check if the provided directory exists + if not os.path.isdir(directory_path): + print(f"Error: The directory '{directory_path}' does not exist.") + sys.exit(1) + + # Process the directory + process_directory(directory_path) diff --git a/core/templates/tree/javascript_react/.eslintrc.cjs b/core/templates/tree/javascript_react/.eslintrc.cjs new file mode 100644 index 00000000..3e212e1d --- /dev/null +++ b/core/templates/tree/javascript_react/.eslintrc.cjs @@ -0,0 +1,21 @@ +module.exports = { + root: true, + env: { browser: true, es2020: true }, + extends: [ + 'eslint:recommended', + 'plugin:react/recommended', + 'plugin:react/jsx-runtime', + 'plugin:react-hooks/recommended', + ], + ignorePatterns: ['dist', '.eslintrc.cjs'], + parserOptions: { ecmaVersion: 'latest', sourceType: 'module' }, + settings: { react: { version: '18.2' } }, + plugins: ['react-refresh'], + rules: { + 'react/jsx-no-target-blank': 'off', + 'react-refresh/only-export-components': [ + 'warn', + { allowConstantExport: true }, + ], + }, +} diff --git a/core/templates/tree/javascript_react/.gitignore b/core/templates/tree/javascript_react/.gitignore new file mode 100644 index 00000000..a547bf36 --- /dev/null +++ b/core/templates/tree/javascript_react/.gitignore @@ -0,0 +1,24 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +lerna-debug.log* + +node_modules +dist +dist-ssr +*.local + +# Editor directories and files +.vscode/* +!.vscode/extensions.json +.idea +.DS_Store +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? diff --git a/core/templates/tree/javascript_react/index.html b/core/templates/tree/javascript_react/index.html new file mode 100644 index 00000000..0777b6a9 --- /dev/null +++ b/core/templates/tree/javascript_react/index.html @@ -0,0 +1,12 @@ + + + + + + {{ project_name }} + + +
+ + + diff --git a/core/templates/tree/javascript_react/package.json b/core/templates/tree/javascript_react/package.json new file mode 100644 index 00000000..54f58642 --- /dev/null +++ b/core/templates/tree/javascript_react/package.json @@ -0,0 +1,26 @@ +{ + "name": "{{ project_name }}", + "private": true, + "version": "0.0.0", + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "lint": "eslint . --ext js,jsx --report-unused-disable-directives --max-warnings 0", + "preview": "vite preview" + }, + "dependencies": { + "react": "^18.2.0", + "react-dom": "^18.2.0" + }, + "devDependencies": { + "@types/react": "^18.2.64", + "@types/react-dom": "^18.2.21", + "@vitejs/plugin-react": "^4.2.1", + "eslint": "^8.57.0", + "eslint-plugin-react": "^7.34.0", + "eslint-plugin-react-hooks": "^4.6.0", + "eslint-plugin-react-refresh": "^0.4.5", + "vite": "^5.1.6" + } +} diff --git a/core/templates/tree/javascript_react/public/.gitkeep b/core/templates/tree/javascript_react/public/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/core/templates/tree/javascript_react/src/App.css b/core/templates/tree/javascript_react/src/App.css new file mode 100644 index 00000000..4014cd73 --- /dev/null +++ b/core/templates/tree/javascript_react/src/App.css @@ -0,0 +1,7 @@ +#root { + max-width: 1280px; + margin: 0 auto; + padding: 2rem; + text-align: center; +} + diff --git a/core/templates/tree/javascript_react/src/App.jsx b/core/templates/tree/javascript_react/src/App.jsx new file mode 100644 index 00000000..f6228cd3 --- /dev/null +++ b/core/templates/tree/javascript_react/src/App.jsx @@ -0,0 +1,11 @@ +import './App.css' + +function App() { + return ( + <> +

{{ project_name }}

+ + ) +} + +export default App diff --git a/core/templates/tree/javascript_react/src/assets/.gitkeep b/core/templates/tree/javascript_react/src/assets/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/core/templates/tree/javascript_react/src/index.css b/core/templates/tree/javascript_react/src/index.css new file mode 100644 index 00000000..7b61cd42 --- /dev/null +++ b/core/templates/tree/javascript_react/src/index.css @@ -0,0 +1,20 @@ +:root { + font-synthesis: none; + text-rendering: optimizeLegibility; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; +} + +body { + margin: 0; + display: flex; + place-items: center; + min-width: 320px; + min-height: 100vh; +} + +h1 { + font-size: 3.2em; + line-height: 1.1; +} + diff --git a/core/templates/tree/javascript_react/src/main.jsx b/core/templates/tree/javascript_react/src/main.jsx new file mode 100644 index 00000000..54b39dd1 --- /dev/null +++ b/core/templates/tree/javascript_react/src/main.jsx @@ -0,0 +1,10 @@ +import React from 'react' +import ReactDOM from 'react-dom/client' +import App from './App.jsx' +import './index.css' + +ReactDOM.createRoot(document.getElementById('root')).render( + + + , +) diff --git a/core/templates/tree/javascript_react/vite.config.js b/core/templates/tree/javascript_react/vite.config.js new file mode 100644 index 00000000..5a33944a --- /dev/null +++ b/core/templates/tree/javascript_react/vite.config.js @@ -0,0 +1,7 @@ +import { defineConfig } from 'vite' +import react from '@vitejs/plugin-react' + +// https://vitejs.dev/config/ +export default defineConfig({ + plugins: [react()], +}) diff --git a/core/templates/tree/node_express_mongoose/.env b/core/templates/tree/node_express_mongoose/.env new file mode 100644 index 00000000..aa7aef74 --- /dev/null +++ b/core/templates/tree/node_express_mongoose/.env @@ -0,0 +1,8 @@ +# Port to listen on (example: 3000) +PORT=3000 + +# MongoDB database URL (example: mongodb://localhost/dbname) +DATABASE_URL=mongodb://localhost/myDb # INPUT_REQUIRED {insert your MongoDB url here} + +# Session secret string (must be unique to your server) +SESSION_SECRET={{ random_secret }} diff --git a/core/templates/tree/node_express_mongoose/.env.example b/core/templates/tree/node_express_mongoose/.env.example new file mode 100644 index 00000000..81ca25c9 --- /dev/null +++ b/core/templates/tree/node_express_mongoose/.env.example @@ -0,0 +1,10 @@ +# Copy this file to .env and edit the settings + +# Port to listen on (example: 3000) +PORT= + +# MongoDB database URL (example: mongodb://localhost/dbname) +DATABASE_URL= + +# Session secret string (must be unique to your server) +SESSION_SECRET= diff --git a/core/templates/tree/node_express_mongoose/models/User.js b/core/templates/tree/node_express_mongoose/models/User.js new file mode 100644 index 00000000..14298001 --- /dev/null +++ b/core/templates/tree/node_express_mongoose/models/User.js @@ -0,0 +1,24 @@ +const mongoose = require('mongoose'); +const bcrypt = require('bcrypt'); + +const userSchema = new mongoose.Schema({ + username: { type: String, unique: true, required: true }, + password: { type: String, required: true } +}); + +userSchema.pre('save', function(next) { + const user = this; + if (!user.isModified('password')) return next(); + bcrypt.hash(user.password, 10, (err, hash) => { + if (err) { + console.error('Error hashing password:', err); + return next(err); + } + user.password = hash; + next(); + }); +}); + +const User = mongoose.model('User', userSchema); + +module.exports = User; \ No newline at end of file diff --git a/core/templates/tree/node_express_mongoose/package.json b/core/templates/tree/node_express_mongoose/package.json new file mode 100644 index 00000000..a382164c --- /dev/null +++ b/core/templates/tree/node_express_mongoose/package.json @@ -0,0 +1,30 @@ +{ + "name": "{{ project_name }}", + "version": "1.0.0", + "description": "", + "main": "server.js", + "scripts": { + "start": "node server.js", + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "dependencies": { + "bcrypt": "^5.1.1", + "body-parser": "^1.20.2", + "chart.js": "^4.4.1", + "connect-flash": "^0.1.1", + "csv-writer": "^1.6.0", + "dotenv": "^16.4.1", + "ejs": "^3.1.9", + "express": "^4.18.2", + "express-session": "^1.18.0", + "connect-mongo": "^5.1.0", + "moment": "^2.30.1", + "mongoose": "^8.1.1", + "axios": "^1.7.7", + "openai": "^4.63.0", + "@anthropic-ai/sdk": "^0.27.3" + } +} diff --git a/core/templates/tree/node_express_mongoose/public/css/style.css b/core/templates/tree/node_express_mongoose/public/css/style.css new file mode 100644 index 00000000..9b3e7fc6 --- /dev/null +++ b/core/templates/tree/node_express_mongoose/public/css/style.css @@ -0,0 +1,17 @@ +/* Placeholder for custom styles */ +body { + padding-bottom: 60px; +} + +footer { + height: 40px; +} + +nav.navbar { + padding: 10px 20px; +} + +.pythagora-logo { + height: 20px; + margin-left: 5px; +} diff --git a/core/templates/tree/node_express_mongoose/public/js/main.js b/core/templates/tree/node_express_mongoose/public/js/main.js new file mode 100644 index 00000000..151865e7 --- /dev/null +++ b/core/templates/tree/node_express_mongoose/public/js/main.js @@ -0,0 +1 @@ +// Placeholder for future JavaScript code diff --git a/core/templates/tree/node_express_mongoose/routes/authRoutes.js b/core/templates/tree/node_express_mongoose/routes/authRoutes.js new file mode 100644 index 00000000..a671fcab --- /dev/null +++ b/core/templates/tree/node_express_mongoose/routes/authRoutes.js @@ -0,0 +1,56 @@ +const express = require('express'); +const User = require('../models/User'); +const bcrypt = require('bcrypt'); +const router = express.Router(); + +router.get('/auth/register', (req, res) => { + res.render('register'); +}); + +router.post('/auth/register', async (req, res) => { + try { + const { username, password } = req.body; + // User model will automatically hash the password using bcrypt + await User.create({ username, password }); + res.redirect('/auth/login'); + } catch (error) { + console.error('Registration error:', error); + res.status(500).send(error.message); + } +}); + +router.get('/auth/login', (req, res) => { + res.render('login'); +}); + +router.post('/auth/login', async (req, res) => { + try { + const { username, password } = req.body; + const user = await User.findOne({ username }); + if (!user) { + return res.status(400).send('User not found'); + } + const isMatch = await bcrypt.compare(password, user.password); + if (isMatch) { + req.session.userId = user._id; + return res.redirect('/'); + } else { + return res.status(400).send('Password is incorrect'); + } + } catch (error) { + console.error('Login error:', error); + return res.status(500).send(error.message); + } +}); + +router.get('/auth/logout', (req, res) => { + req.session.destroy(err => { + if (err) { + console.error('Error during session destruction:', err); // gpt_pilot_debugging_log + return res.status(500).send('Error logging out'); + } + res.redirect('/auth/login'); + }); +}); + +module.exports = router; diff --git a/core/templates/tree/node_express_mongoose/routes/middleware/authMiddleware.js b/core/templates/tree/node_express_mongoose/routes/middleware/authMiddleware.js new file mode 100644 index 00000000..8f884a8f --- /dev/null +++ b/core/templates/tree/node_express_mongoose/routes/middleware/authMiddleware.js @@ -0,0 +1,20 @@ +const User = require('../../models/User'); +const isAuthenticated = async (req, res, next) => { + if (req.session && req.session.userId) { + try { + const user = await User.findById(req.session.userId); + if (user) { + req.user = user; + return next(); + } + } catch (error) { + console.error('Error in authentication middleware:', error); + res.status(500).send('Error during authentication process'); + } + } + return res.status(401).send('You are not authenticated'); +}; + +module.exports = { + isAuthenticated +}; diff --git a/core/templates/tree/node_express_mongoose/server.js b/core/templates/tree/node_express_mongoose/server.js new file mode 100644 index 00000000..0cea71cf --- /dev/null +++ b/core/templates/tree/node_express_mongoose/server.js @@ -0,0 +1,93 @@ +// Load environment variables +require("dotenv").config(); +const mongoose = require("mongoose"); +const express = require("express"); +const session = require("express-session"); +const MongoStore = require('connect-mongo'); +const authRoutes = require("./routes/authRoutes"); + +if (!process.env.DATABASE_URL || !process.env.SESSION_SECRET) { + console.error("Error: config environment variables not set. Please create/edit .env configuration file."); + process.exit(-1); +} + +const app = express(); +const port = process.env.PORT || 3000; + +// Middleware to parse request bodies +app.use(express.urlencoded({ extended: true })); +app.use(express.json()); + +// Setting the templating engine to EJS +app.set("view engine", "ejs"); + +// Serve static files +app.use(express.static("public")); + +// Database connection +mongoose + .connect(process.env.DATABASE_URL) + .then(() => { + console.log("Database connected successfully"); + }) + .catch((err) => { + console.error(`Database connection error: ${err.message}`); + console.error(err.stack); + process.exit(1); + }); + +// Session configuration with connect-mongo +app.use( + session({ + secret: process.env.SESSION_SECRET, + resave: false, + saveUninitialized: false, + store: MongoStore.create({ mongoUrl: process.env.DATABASE_URL }), + }), +); + +app.on("error", (error) => { + console.error(`Server error: ${error.message}`); + console.error(error.stack); +}); + +// Logging session creation and destruction +app.use((req, res, next) => { + const sess = req.session; + // Make session available to all views + res.locals.session = sess; + if (!sess.views) { + sess.views = 1; + console.log("Session created at: ", new Date().toISOString()); + } else { + sess.views++; + console.log( + `Session accessed again at: ${new Date().toISOString()}, Views: ${sess.views}, User ID: ${sess.userId || '(unauthenticated)'}`, + ); + } + next(); +}); + +// Authentication Routes +app.use(authRoutes); + +// Root path response +app.get("/", (req, res) => { + res.render("index"); +}); + +// If no routes handled the request, it's a 404 +app.use((req, res, next) => { + res.status(404).send("Page not found."); +}); + +// Error handling +app.use((err, req, res, next) => { + console.error(`Unhandled application error: ${err.message}`); + console.error(err.stack); + res.status(500).send("There was an error serving your request."); +}); + +app.listen(port, () => { + console.log(`Server running at http://localhost:${port}`); +}); diff --git a/core/templates/tree/node_express_mongoose/services/llm.js b/core/templates/tree/node_express_mongoose/services/llm.js new file mode 100644 index 00000000..9b8fe4cc --- /dev/null +++ b/core/templates/tree/node_express_mongoose/services/llm.js @@ -0,0 +1,72 @@ +const axios = require('axios'); +const OpenAI = require('openai'); +const Anthropic = require('@anthropic-ai/sdk'); +const dotenv = require('dotenv'); + +dotenv.config(); + +const openai = new OpenAI({ + apiKey: process.env.OPENAI_API_KEY, +}); + +const anthropic = new Anthropic({ + apiKey: process.env.ANTHROPIC_API_KEY, +}); + +const MAX_RETRIES = 3; +const RETRY_DELAY = 1000; + +async function sleep(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +async function sendRequestToOpenAI(model, message) { + for (let i = 0; i < MAX_RETRIES; i++) { + try { + const response = await openai.chat.completions.create({ + model: model, + messages: [{ role: 'user', content: message }], + max_tokens: 1024, + }); + return response.choices[0].message.content; + } catch (error) { + console.error(`Error sending request to OpenAI (attempt ${i + 1}):`, error.message, error.stack); + if (i === MAX_RETRIES - 1) throw error; + await sleep(RETRY_DELAY); + } + } +} + +async function sendRequestToAnthropic(model, message) { + for (let i = 0; i < MAX_RETRIES; i++) { + try { + console.log(`Sending request to Anthropic with model: ${model} and message: ${message}`); + const response = await anthropic.messages.create({ + model: model, + messages: [{ role: 'user', content: message }], + max_tokens: 1024, + }); + console.log(`Received response from Anthropic: ${JSON.stringify(response.content)}`); + return response.content[0].text; + } catch (error) { + console.error(`Error sending request to Anthropic (attempt ${i + 1}):`, error.message, error.stack); + if (i === MAX_RETRIES - 1) throw error; + await sleep(RETRY_DELAY); + } + } +} + +async function sendLLMRequest(provider, model, message) { + switch (provider.toLowerCase()) { + case 'openai': + return sendRequestToOpenAI(model, message); + case 'anthropic': + return sendRequestToAnthropic(model, message); + default: + throw new Error(`Unsupported LLM provider: ${provider}`); + } +} + +module.exports = { + sendLLMRequest +}; diff --git a/core/templates/tree/node_express_mongoose/views/index.ejs b/core/templates/tree/node_express_mongoose/views/index.ejs new file mode 100644 index 00000000..ac5f6c94 --- /dev/null +++ b/core/templates/tree/node_express_mongoose/views/index.ejs @@ -0,0 +1,14 @@ + + +<%- include('partials/_head.ejs') %> + +<%- include('partials/_header.ejs') %> +
+
+

{{ project_name }}

+
+
+ +<%- include('partials/_footer.ejs') %> + + diff --git a/core/templates/tree/node_express_mongoose/views/login.ejs b/core/templates/tree/node_express_mongoose/views/login.ejs new file mode 100644 index 00000000..a3e3be56 --- /dev/null +++ b/core/templates/tree/node_express_mongoose/views/login.ejs @@ -0,0 +1,25 @@ + + +<%- include('partials/_head.ejs') %> + +<%- include('partials/_header.ejs') %> +
+
+

Login

+
+
+ +
+
+ +
+
+ + Don't have an account? Register +
+
+
+
+ +<%- include('partials/_footer.ejs') %> + diff --git a/core/templates/tree/node_express_mongoose/views/partials/_footer.ejs b/core/templates/tree/node_express_mongoose/views/partials/_footer.ejs new file mode 100644 index 00000000..0e1a76aa --- /dev/null +++ b/core/templates/tree/node_express_mongoose/views/partials/_footer.ejs @@ -0,0 +1,8 @@ +
+
+ Built with + +
+
+ + diff --git a/core/templates/tree/node_express_mongoose/views/partials/_head.ejs b/core/templates/tree/node_express_mongoose/views/partials/_head.ejs new file mode 100644 index 00000000..807f3eb1 --- /dev/null +++ b/core/templates/tree/node_express_mongoose/views/partials/_head.ejs @@ -0,0 +1,7 @@ + + + {{ project_name }} + + + + diff --git a/core/templates/tree/node_express_mongoose/views/partials/_header.ejs b/core/templates/tree/node_express_mongoose/views/partials/_header.ejs new file mode 100644 index 00000000..612b9a94 --- /dev/null +++ b/core/templates/tree/node_express_mongoose/views/partials/_header.ejs @@ -0,0 +1,20 @@ + diff --git a/core/templates/tree/node_express_mongoose/views/register.ejs b/core/templates/tree/node_express_mongoose/views/register.ejs new file mode 100644 index 00000000..c42448f3 --- /dev/null +++ b/core/templates/tree/node_express_mongoose/views/register.ejs @@ -0,0 +1,25 @@ + + +<%- include('partials/_head.ejs') %> + +<%- include('partials/_header.ejs') %> +
+
+

Register

+
+
+ +
+
+ +
+
+ + Already have an account? Login +
+
+
+
+ +<%- include('partials/_footer.ejs') %> + diff --git a/core/templates/tree/react_express/.babelrc b/core/templates/tree/react_express/.babelrc new file mode 100644 index 00000000..6d1e60a2 --- /dev/null +++ b/core/templates/tree/react_express/.babelrc @@ -0,0 +1,13 @@ +{ + "presets": [ + [ + "env", + { + "targets": { + "node": "current" + } + } + ], + "jest" + ] +} diff --git a/core/templates/tree/react_express/.env b/core/templates/tree/react_express/.env new file mode 100644 index 00000000..b50bd841 --- /dev/null +++ b/core/templates/tree/react_express/.env @@ -0,0 +1,44 @@ +# Node environment - "production" (default) or "development" +NODE_ENV=development + +# Log level to use, default "info" in production and "debug" in development +# See https://github.com/pinojs/pino +LOG_LEVEL=debug + +# Port to listen on, default 3000 +PORT=3000 + +{% if options.db_type == 'sql' %} +# See https://www.prisma.io/docs/reference/database-reference/connection-urls#format +# For PostgreSQL: +# DATABASE_PROVIDER=postgresql +# DATABASE_URL=postgresql://user:password@host/database +# Default is SQLite: +DATABASE_PROVIDER=sqlite +DATABASE_URL=file:./db.sqlite + +{% elif options.db_type == 'nosql' %} +# See https://mongoosejs.com/docs/connections.html +DATABASE_URL=mongodb://localhost/myDb + +{% endif %} +{% if options.email %} +# E-mail sending with nodemailer; see https://nodemailer.com/smtp/#general-options +# NODEMAILER_HOST= +# NODEMAILER_PORT=25 +# NODEMAILER_USER= +# NODEMAILER_PASS= +# NODEMAILER_SECURE=false + +{% endif %} +{% if options.bg_tasks %} +# URL pointing to Redis, default is redis://127.0.0.1:6379 (localhost) +# REDIS_URL= + +# Queue name for background tasks using bull +# BG_TASKS_QUEUE=bg-tasks + +{% endif %} +# Session secret string (must be unique to your server) +SESSION_SECRET={{ random_secret }} + diff --git a/core/templates/tree/react_express/.eslintrc.json b/core/templates/tree/react_express/.eslintrc.json new file mode 100644 index 00000000..28974586 --- /dev/null +++ b/core/templates/tree/react_express/.eslintrc.json @@ -0,0 +1,18 @@ +{ + "env": { + "browser": true, + "es2021": true, + "node": true, + "jest": true + }, + "extends": [ + "eslint:recommended" + ], + "parserOptions": { + "ecmaVersion": 12, + "sourceType": "module" + }, + "rules": { + "no-unused-vars": ["error", { "argsIgnorePattern": "(req|res|next)" }] + } +} diff --git a/core/templates/tree/react_express/.gitignore b/core/templates/tree/react_express/.gitignore new file mode 100644 index 00000000..199305c2 --- /dev/null +++ b/core/templates/tree/react_express/.gitignore @@ -0,0 +1,32 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +lerna-debug.log* + +node_modules +dist +dist-ssr +*.local + +# Editor directories and files +.vscode/* +!.vscode/extensions.json +.idea +.DS_Store +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? + +# SQLite databases, data files +*.db +*.csv + +# Keep environment variables out of version control +.env + diff --git a/core/templates/tree/react_express/README.md b/core/templates/tree/react_express/README.md new file mode 100644 index 00000000..e4b23f1c --- /dev/null +++ b/core/templates/tree/react_express/README.md @@ -0,0 +1,92 @@ +# {{ project_name }} + +{{ project_description }} + +## Quickstart + +1. Install required packages: + + ``` + npm install + ``` + +2. Update `.env` with your settings. + +{% if options.db_type == 'sql' %} +3. Create initial database migration: + + ``` + npx prisma migrate dev --name initial + ``` + + When run the first time, it will also install + `@prisma/client` and generate client code. + +4. Run the tests: +{% else %} +3. Run the tests: +{% endif %} + + ``` + npm run test + ``` + +## Development + +To run the server in development mode, with log pretty-printing and hot-reload: + +``` +npm run dev +``` + +To run the tests, run the `test` script (`npm run test`). ESLint is used for linting and its configuration is specified in `.eslintrc.json`. + +Code style is automatically formatted using `prettier`. To manually run prettier, use `npm run prettier`. Better yet, integrate your editor to run it on save. + +## Production + +To run the app in production, run: + +``` +npm start +``` + +Logs will be sent to the standard output in JSON format. +{% if options.bg_tasks %} + +## Background tasks with Bull + +A simple task queue is built using `bull` and backed by Redis. Tasks are defined and exported in `src/tasks.js`. Call proxies are created automatically and tasks can be queued with: + +``` +import { tasks } from "./src/utils/queue.js"; +const result = await tasks.someFunction(...); +``` + +To run the worker(s) that will execute the queued tasks, run: + +``` +npm run worker +``` +{% endif %} + +## Using Docker + +Build the docker image with: + + docker build -t {{ project_folder }} . + +The default command is to start the web server (gunicorn). Run the image with `-P` docker option to expose the internal port (3000) and check the exposed port with `docker ps`: + + docker run --env-file .env --P {{ project_folder }} + docker ps + +Make sure you provide the correct path to the env file (this example assumes it's located in the local directory). + +To run a custom command using the image (for example, starting the Node +shell): + + docker run --env-file .env {{ project_folder }} npm run shell + +For more information on the docker build process, see the included +`Dockerfile`. diff --git a/core/templates/tree/react_express/api/app.js b/core/templates/tree/react_express/api/app.js new file mode 100644 index 00000000..4c135d1b --- /dev/null +++ b/core/templates/tree/react_express/api/app.js @@ -0,0 +1,56 @@ +import path from 'path'; +import { existsSync } from 'fs'; +import { fileURLToPath } from 'node:url'; + +import cors from 'cors'; +import express from 'express'; + +{% if options.auth %} +import authRoutes from './routes/authRoutes.js'; +import { authenticateWithToken } from './middlewares/authMiddleware.js'; +{% endif %} +import apiRoutes from './routes/index.js'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +// Set up Express app +const app = express(); + +// Pretty-print JSON responses +app.enable('json spaces'); +// We want to be consistent with URL paths, so we enable strict routing +app.enable('strict routing'); + +app.use(express.json()); +app.use(express.urlencoded({ extended: true })); +app.use(cors()); +{% if options.auth %} + +// Authentication routes +app.use(authenticateWithToken); +app.use(authRoutes); +{% endif %} + +app.use(apiRoutes); + +app.use(express.static(path.join(__dirname, "..", "dist"))); + +// Assume all other routes are frontend +app.get(/.*/, async (req, res) => { + // Try to serve pre-built frontend from ../dist/ folder + const clientBundlePath = path.join(__dirname, "..", "dist", "index.html"); + + if (!existsSync(clientBundlePath)) { + if (process.env.NODE_ENV === "development") { + // In development, we just want to redirect to the Vite dev server + return res.redirect("http://localhost:5173"); + } else { + // Looks like "npm run build:ui" wasn't run and the UI isn't built, show a nice error message instead + return res.status(404).send("Front-end not available."); + } + } + res.sendFile(path.join(import.meta.dirname, "..", "dist", "index.html")); +}); + +export default app; diff --git a/core/templates/tree/react_express/api/middlewares/authMiddleware.js b/core/templates/tree/react_express/api/middlewares/authMiddleware.js new file mode 100644 index 00000000..cba76c28 --- /dev/null +++ b/core/templates/tree/react_express/api/middlewares/authMiddleware.js @@ -0,0 +1,29 @@ +import UserService from '../services/userService.js'; + +export const authenticateWithToken = (req, res, next) => { + const authHeader = req.get('Authorization'); + if (authHeader) { + const m = authHeader.match(/^(Token|Bearer) (.+)/i); + if (m) { + UserService.authenticateWithToken(m[2]) + .then((user) => { + req.user = user; + next(); + }) + .catch((err) => { + next(err); + }); + return; + } + } + + next(); +}; + +export const requireUser = (req, res, next) => { + if (!req.user) { + return res.status(401).json({ error: 'Authentication required' }); + } + + next(); +}; diff --git a/core/templates/tree/react_express/api/middlewares/errorMiddleware.js b/core/templates/tree/react_express/api/middlewares/errorMiddleware.js new file mode 100644 index 00000000..844b0e37 --- /dev/null +++ b/core/templates/tree/react_express/api/middlewares/errorMiddleware.js @@ -0,0 +1,22 @@ +import logger from '../utils/log.js'; + +const log = logger('api:middleware'); + +/* 404 handler for the missing API endpoints + * Due to how Express works, we don't know if the URL or HTTP method is + * incorrect, so we return 404 in both cases. + */ +export const handle404 = (req, res, next) => { + const { method, originalUrl } = req; + log.info({ method, originalUrl }, `Unhandled API request ${method} ${originalUrl}`); + return res.status(404).json({ error: 'Resource not found or unsupported HTTP method' }); +}; + +/* 500 handler in case we have an error in one of our route handlers + */ +export const handleError = (error, req, res, next) => { + const { method, originalUrl } = req; + + log.error({ method, originalUrl, error }, `Error while handling ${method} ${originalUrl}`); + res.status(500).json({ error }); +}; diff --git a/core/templates/tree/react_express/api/models/init.js b/core/templates/tree/react_express/api/models/init.js new file mode 100644 index 00000000..28b8c8de --- /dev/null +++ b/core/templates/tree/react_express/api/models/init.js @@ -0,0 +1,31 @@ +{% if options.db_type == 'nosql' %} +import mongoose from 'mongoose'; +import logger from '../utils/log.js'; + +const log = logger('models'); + +const dbInit = async (options = {}) => { + const mongoUrl = process.env.DATABASE_URL || 'mongodb://localhost/myDb'; + + try { + await mongoose.connect(mongoUrl, options); + log.debug(`Connected to MongoDB at ${mongoUrl}`); + } catch (err) { + log.fatal(`Error connecting to database ${mongoUrl}:`, err); + throw err; + } +}; + +export default dbInit; +{% endif %} +{% if options.db_type == 'sql' %} +import Prisma from '@prisma/client'; + +// PrismaClient is not available when testing +const { PrismaClient } = Prisma || {}; +const prisma = PrismaClient ? new PrismaClient() : {}; + +{% if options.auth %} +export const User = prisma.user; +{% endif %} +{% endif %} diff --git a/core/templates/tree/react_express/api/models/user.js b/core/templates/tree/react_express/api/models/user.js new file mode 100644 index 00000000..cb091ec9 --- /dev/null +++ b/core/templates/tree/react_express/api/models/user.js @@ -0,0 +1,79 @@ +import mongoose from 'mongoose'; +import { randomUUID } from 'crypto'; +import isEmail from 'validator/lib/isEmail.js'; + +import { generatePasswordHash, validatePassword, isPasswordHash } from '../utils/password.js'; + +const schema = new mongoose.Schema({ + email: { + type: String, + required: true, + index: true, + unique: true, + lowercase: true, + validate: { validator: isEmail, message: 'Invalid email' }, + }, + password: { + type: String, + required: true, + validate: { validator: isPasswordHash, message: 'Invalid password hash' }, + }, + token: { + type: String, + unique: true, + index: true, + default: () => randomUUID(), + }, + name: { + type: String, + }, + createdAt: { + type: Date, + default: Date.now, + immutable: true, + }, + lastLoginAt: { + type: Date, + default: Date.now, + }, + isActive: { + type: Boolean, + default: true, + }, +}, { + versionKey: false, +}); + +schema.set('toJSON', { + /* eslint-disable */ + transform: (doc, ret, options) => { + delete ret._id; + delete ret.password; + return ret; + }, + /* eslint-enable */ +}); + +schema.statics.authenticateWithPassword = async function authenticateWithPassword(email, password) { + const user = await this.findOne({ email }).exec(); + if (!user) return null; + + const passwordValid = await validatePassword(password, user.password); + if (!passwordValid) return null; + + user.lastLoginAt = Date.now(); + const updatedUser = await user.save(); + + return updatedUser; +}; + +schema.methods.regenerateToken = async function regenerateToken() { + this.token = randomUUID(); + if (!this.isNew) { + await this.save(); + } + return this; +}; + +const User = mongoose.model('User', schema); +export default User; diff --git a/core/templates/tree/react_express/api/routes/authRoutes.js b/core/templates/tree/react_express/api/routes/authRoutes.js new file mode 100644 index 00000000..d0956748 --- /dev/null +++ b/core/templates/tree/react_express/api/routes/authRoutes.js @@ -0,0 +1,68 @@ +import { Router } from 'express'; + +import UserService from '../services/userService.js'; +import { requireUser } from '../middlewares/authMiddleware.js'; +import logger from '../utils/log.js'; + +const log = logger('api/routes/authRoutes'); + +const router = Router(); + +router.post('/api/auth/login', async (req, res) => { + const sendError = msg => res.status(400).json({ error: msg }); + const { email, password } = req.body; + + if (!email || !password) { + return sendError('Email and password are required'); + } + + const user = await UserService.authenticateWithPassword(email, password); + + if (user) { + return res.json(user); + } else { + return sendError('Email or password is incorrect'); + + } +}); + +router.get('/api/auth/login', (req, res) => res.status(405).json({ error: 'Login with POST instead' })); + +router.post('/api/auth/register', async (req, res, next) => { + if (req.user) { + return res.json({ user: req.user }); + } + try { + const user = await UserService.createUser(req.body); + return res.status(201).json(user); + } catch (error) { + log.error('Error while registering user', error); + return res.status(400).json({ error }); + } +}); + +router.get('/api/auth/register', (req, res) => res.status(405).json({ error: 'Register with POST instead' })); + +router.all('/api/auth/logout', async (req, res) => { + if (req.user) { + await UserService.regenerateToken(req.user); + } + return res.status(204).send(); +}); + +router.post('/api/auth/password', requireUser, async (req, res) => { + const { password } = req.body; + + if (!password) { + return res.status(400).json({ error: 'Password is required' }); + } + + await UserService.setPassword(req.user, password); + res.status(204).send(); +}); + +router.get('/api/auth/me', requireUser, async (req, res) => { + return res.status(200).json(req.user); +}); + +export default router; diff --git a/core/templates/tree/react_express/api/routes/index.js b/core/templates/tree/react_express/api/routes/index.js new file mode 100644 index 00000000..39cc638e --- /dev/null +++ b/core/templates/tree/react_express/api/routes/index.js @@ -0,0 +1,8 @@ +import { Router } from 'express'; + +const router = Router(); + +// Define API routes here +// All API routes must have /api/ prefix to avoid conflicts with the UI/frontend. + +export default router; \ No newline at end of file diff --git a/core/templates/tree/react_express/api/services/userService.js b/core/templates/tree/react_express/api/services/userService.js new file mode 100644 index 00000000..70c869d0 --- /dev/null +++ b/core/templates/tree/react_express/api/services/userService.js @@ -0,0 +1,232 @@ +import { randomUUID } from 'crypto'; + +{% set mongoose = options.db_type == 'nosql' %} +{% if mongoose %} +import User from '../models/user.js'; +{% else %} +import { User } from '../models/init.js'; +{% endif %} +import { generatePasswordHash, validatePassword } from '../utils/password.js'; + +class UserService { + static async list() { + try { +{% if mongoose %} + return User.find(); +{% else %} + const users = await User.findMany(); + return users.map((u) => ({ ...u, password: undefined })); +{% endif %} + } catch (err) { + throw `Database error while listing users: ${err}`; + } + } + + static async get(id) { + try { +{% if mongoose %} + return User.findOne({ _id: id }).exec(); +{% else %} + const user = await User.findUnique({ + where: { id }, + }); + + if (!user) return null; + + delete user.password; + return user; +{% endif %} + } catch (err) { + throw `Database error while getting the user by their ID: ${err}`; + } + } + + static async getByEmail(email) { + try { +{% if mongoose %} + return User.findOne({ email }).exec(); +{% else %} + const user = await User.findUnique({ + where: { email }, + }); + + if (!user) return null; + + delete user.password; + return user; +{% endif %} + } catch (err) { + throw `Database error while getting the user by their email: ${err}`; + } + } + + static async update(id, data) { + try { +{% if mongoose %} + return User.findOneAndUpdate({ _id: id }, data, { new: true, upsert: false }); +{% else %} + return User.update({ + where: { id }, + }, { + data, + }); +{% endif %} + } catch (err) { + throw `Database error while updating user ${id}: ${err}`; + } + } + + static async delete(id) { + try { +{% if mongoose %} + const result = await User.deleteOne({ _id: id }).exec(); + return (result.deletedCount === 1); +{% else %} + return User.delete({ + where: { id }, + }); +{% endif %} + } catch (err) { + throw `Database error while deleting user ${id}: ${err}`; + } + } + + static async authenticateWithPassword(email, password) { + if (!email) throw 'Email is required'; + if (!password) throw 'Password is required'; + + try { +{% if mongoose %} + const user = await User.findOne({email}).exec(); +{% else %} + const user = await User.findUnique({ + where: {email}, + }); +{% endif %} + if (!user) return null; + + const passwordValid = await validatePassword(password, user.password); + if (!passwordValid) return null; + +{% if mongoose %} + user.lastLoginAt = Date.now(); + const updatedUser = await user.save(); +{% else %} + user.lastLoginAt = new Date(); + const updatedUser = await User.update({ + where: { id: user.id }, + data: { lastLoginAt: user.lastLoginAt }, + }); + + delete updatedUser.password; +{% endif %} + return updatedUser; + } catch (err) { + throw `Database error while authenticating user ${email} with password: ${err}`; + } + } + + static async authenticateWithToken(token) { + try { +{% if mongoose %} + return User.findOne({ token }).exec(); +{% else %} + const user = await User.findUnique({ + where: { token }, + }); + if (!user) return null; + + delete user.password; + return user; +{% endif %} + } catch (err) { + throw `Database error while authenticating user ${email} with token: ${err}`; + } + } + + static async createUser({ email, password, name = '' }) { + if (!email) throw 'Email is required'; + if (!password) throw 'Password is required'; + + const existingUser = await UserService.getByEmail(email); + if (existingUser) throw 'User with this email already exists'; + + const hash = await generatePasswordHash(password); + + try { +{% if mongoose %} + const user = new User({ + email, + password: hash, + name, + token: randomUUID(), + }); + + await user.save(); +{% else %} + const data = { + email, + password: hash, + name, + token: randomUUID(), + }; + + const user = await User.create({ data }); + + delete user.password; +{% endif %} + return user; + } catch (err) { + throw `Database error while creating new user: ${err}`; + } + } + + static async setPassword(user, password) { + if (!password) throw 'Password is required'; + user.password = await generatePasswordHash(password); // eslint-disable-line + + try { +{% if mongoose %} + if (!user.isNew) { + await user.save(); + } +{% else %} + if (user.id) { + return User.update({ + where: { id: user.id }, + data: { password: user.password }, + }); + } +{% endif %} + + return user; + } catch (err) { + throw `Database error while setting user password: ${err}`; + } + } + + static async regenerateToken(user) { + user.token = randomUUID(); // eslint-disable-line + + try { +{% if mongoose %} + if (!user.isNew) { + await user.save(); + } +{% else %} + if (user.id) { + return User.update({ + where: { id: user.id }, + data: { password: user.password }, + }); + } +{% endif %} + + return user; + } catch (err) { + throw `Database error while generating user token: ${err}`; + } + } +} + +export default UserService; diff --git a/core/templates/tree/react_express/api/utils/log.js b/core/templates/tree/react_express/api/utils/log.js new file mode 100644 index 00000000..464e6507 --- /dev/null +++ b/core/templates/tree/react_express/api/utils/log.js @@ -0,0 +1,13 @@ +import pino from 'pino'; + +const DEFAULT_LOG_LEVEL = process.env.NODE_ENV === "production" ? "info" : "debug"; +const level = process.env.LOG_LEVEL || DEFAULT_LOG_LEVEL; + +if (!pino.levels.values[level]) { + const validLevels = Object.keys(pino.levels.values).join(', '); + throw new Error(`Log level must be one of: ${validLevels}`); +} + +const logger = (name) => pino({ name, level }); + +export default logger; diff --git a/core/templates/tree/react_express/api/utils/mail.js b/core/templates/tree/react_express/api/utils/mail.js new file mode 100644 index 00000000..611b1ed2 --- /dev/null +++ b/core/templates/tree/react_express/api/utils/mail.js @@ -0,0 +1,33 @@ +/* Send mail using nodemailer + * + * Configure using NODEMAILER_* env variables. + * See https://nodemailer.com/smtp/ for all options + * + * Send mail with: + * + * import transport from "./src/utils/mail.js"; + * await transport.sendMail({ from, to, subject, text }); + * + * For all message options, see: https://nodemailer.com/message/ + */ +import nodemailer from "nodemailer"; + +import config from "./config.js"; + +const options = { + host: config.NODEMAILER_HOST, + port: config.NODEMAILER_PORT, + secure: config.NODEMAILER_SECURE, +}; + +if (config.NODEMAILER_USER && config.NODMAILER_PASS) { + options.auth = { + user: config.NODEMAILER_USER, + pass: config.NODEMAILER_PASS, + }; +} + +const transporter = nodemailer.createTransport(options); +const sendMail = transporter.sendMail.bind(transporter); + +export default sendMail; diff --git a/core/templates/tree/react_express/api/utils/password.js b/core/templates/tree/react_express/api/utils/password.js new file mode 100644 index 00000000..a38bd4a1 --- /dev/null +++ b/core/templates/tree/react_express/api/utils/password.js @@ -0,0 +1,38 @@ +import bcrypt from 'bcrypt'; + +/** + * Hashes the password using bcrypt algorithm + * @param {string} password - The password to hash + * @return {string} Password hash + */ +export const generatePasswordHash = async (password) => { + const salt = await bcrypt.genSalt(); + const hash = await bcrypt.hash(password, salt); + return hash; +}; + +/** + * Validates the password against the hash + * @param {string} password - The password to verify + * @param {string} hash - Password hash to verify against + * @return {boolean} True if the password matches the hash, false otherwise + */ +export const validatePassword = async (password, hash) => { + const result = await bcrypt.compare(password, hash); + return result; +}; + +/** + * Checks that the hash has a valid format + * @param {string} hash - Hash to check format for + * @return {boolean} True if passed string seems like valid hash, false otherwise + */ +export const isPasswordHash = (hash) => { + if (!hash || hash.length !== 60) return false; + try { + bcrypt.getRounds(hash); + return true; + } catch { + return false; + } +}; diff --git a/core/templates/tree/react_express/components.json b/core/templates/tree/react_express/components.json new file mode 100644 index 00000000..b887755a --- /dev/null +++ b/core/templates/tree/react_express/components.json @@ -0,0 +1,17 @@ +{ + "$schema": "https://ui.shadcn.com/schema.json", + "style": "default", + "rsc": false, + "tsx": false, + "tailwind": { + "config": "tailwind.config.js", + "css": "ui/index.css", + "baseColor": "slate", + "cssVariables": true, + "prefix": "" + }, + "aliases": { + "components": "@/components", + "utils": "@/lib/utils" + } +} diff --git a/core/templates/tree/react_express/index.html b/core/templates/tree/react_express/index.html new file mode 100644 index 00000000..23d6c3f2 --- /dev/null +++ b/core/templates/tree/react_express/index.html @@ -0,0 +1,15 @@ + + + + + + + {{ project_name }} + + + +
+ + + + \ No newline at end of file diff --git a/core/templates/tree/react_express/jsconfig.json b/core/templates/tree/react_express/jsconfig.json new file mode 100644 index 00000000..dc1da568 --- /dev/null +++ b/core/templates/tree/react_express/jsconfig.json @@ -0,0 +1,10 @@ +{ + "compilerOptions": { + "baseUrl": ".", + "paths": { + "@/*": [ + "./ui/*" + ] + } + } +} diff --git a/core/templates/tree/react_express/package.json b/core/templates/tree/react_express/package.json new file mode 100644 index 00000000..d331f301 --- /dev/null +++ b/core/templates/tree/react_express/package.json @@ -0,0 +1,77 @@ +{ + "name": "{{ folder_name }}", + "type": "module", + "version": "0.0.1", + "description": "{{ project_name|escape_string}}", + "main": "server.js", + "author": "", + "license": "UNLICENSED", + "dependencies": { + "axios": "^1.7.2", + "@radix-ui/react-label": "^2.1.0", + "@radix-ui/react-slot": "^1.1.0", + "bcrypt": "*", + "bull": "*", + "class-variance-authority": "^0.7.0", + "clsx": "^2.1.1", + "cors": "*", + "dotenv": "*", + "express": "*", + "jsonschema": "*", +{% if options.db_type == 'nosql' %} + "mongoose": "*", + "validator": "*", +{% endif %} + "lucide-react": "^0.395.0", + "nodemailer": "*", + "pino": "*", + "tailwind-merge": "^2.3.0", + "tailwindcss-animate": "^1.0.7" + }, + "devDependencies": { + "@prisma/client": "*", + "@types/node": "^20.14.6", + "@types/react": "^18.2.64", + "@types/react-dom": "^18.2.21", + "@vitejs/plugin-react": "^4.2.1", + "autoprefixer": "^10.4.19", + "babel": "*", + "babel-preset-env": "*", + "babel-preset-jest": "*", + "concurrently": "*", + "eslint": "*", + "eslint-config-airbnb-base": "*", + "eslint-plugin-import": "*", + "eslint-plugin-react": "^7.34.0", + "eslint-plugin-react-hooks": "^4.6.0", + "eslint-plugin-react-refresh": "^0.4.5", + "jest": "*", + "nodemon": "*", + "pino-pretty": "*", + "postcss": "^8.4.38", + "prettier": "*", +{% if options.db_type == 'sql' %} + "prisma": "*", +{% endif %} + "react": "^18.2.0", + "react-dom": "^18.2.0", + "react-router-dom": "*", + "supertest": "*", + "tailwindcss": "^3.4.4", + "vite": "^5.1.6" + }, + "scripts": { + "start:api": "node server.js", + "dev:api": "nodemon -w api -w .env -w server.js server | pino-pretty -clt -i 'hostname,pid'", + "lint:api": "eslint .", + "prettier:api": "prettier -w .", + "test:api": "jest --roots test --verbose", + "coverage:api": "jest --roots test --verbose --coverage", + "watch-test:api": "jest --roots test --verbose --watch", + "dev:ui": "vite", + "build:ui": "vite build", + "lint:ui": "eslint . --ext js,jsx --report-unused-disable-directives --max-warnings 0", + "preview:ui": "vite preview", + "dev": "concurrently -n api,ui \"npm:dev:api\" \"npm:dev:ui\"" + } +} diff --git a/core/templates/tree/react_express/postcss.config.js b/core/templates/tree/react_express/postcss.config.js new file mode 100644 index 00000000..2e7af2b7 --- /dev/null +++ b/core/templates/tree/react_express/postcss.config.js @@ -0,0 +1,6 @@ +export default { + plugins: { + tailwindcss: {}, + autoprefixer: {}, + }, +} diff --git a/core/templates/tree/react_express/prisma/schema.prisma b/core/templates/tree/react_express/prisma/schema.prisma new file mode 100644 index 00000000..b728cda4 --- /dev/null +++ b/core/templates/tree/react_express/prisma/schema.prisma @@ -0,0 +1,27 @@ +// Prisma schema file +// See https://www.prisma.io/docs/concepts/components/prisma-schema + +datasource db { + provider = "sqlite" + url = env("DATABASE_URL") +} + +generator client { + provider = "prisma-client-js" +} +{% if options.auth %} + +model User { + id Int @id @default(autoincrement()) + email String @unique + password String + token String @unique + name String + createdAt DateTime @default(now()) + lastLoginAt DateTime @default(now()) + isActive Boolean @default(true) + + @@index([email]) + @@index([token]) +} +{% endif %} diff --git a/core/templates/tree/react_express/public/.gitkeep b/core/templates/tree/react_express/public/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/core/templates/tree/react_express/server.js b/core/templates/tree/react_express/server.js new file mode 100644 index 00000000..33d9d263 --- /dev/null +++ b/core/templates/tree/react_express/server.js @@ -0,0 +1,34 @@ +import http from 'http'; +import dotenv from 'dotenv'; + +dotenv.config(); + +import app from './api/app.js'; +import logger from './api/utils/log.js'; +{% if options.db_type == 'nosql' %} +import mongoInit from './api/models/init.js'; +{% endif %} + +const log = logger('server'); +const server = http.createServer(app); + +process.on('uncaughtException', (err) => { + log.fatal({ err }, `Unhandled error ${err}`); + server.close(); +}); + +process.on('unhandledRejection', (reason) => { + log.error(`Unhandled error (in promise): ${reason}`); +}); + +// Main entry point to the application +const main = async () => { + {% if options.db_type == 'nosql' %} + await mongoInit(); + {% endif %} + const port = parseInt(process.env.PORT) || 3000; + log.info(`Listening on http://localhost:${port}/`); + await server.listen(port); +}; + +main(); diff --git a/core/templates/tree/react_express/tailwind.config.js b/core/templates/tree/react_express/tailwind.config.js new file mode 100644 index 00000000..247f47ee --- /dev/null +++ b/core/templates/tree/react_express/tailwind.config.js @@ -0,0 +1,76 @@ +/** @type {import('tailwindcss').Config} */ +module.exports = { + darkMode: ["class"], + content: [ + './ui/main.jsx', + './ui/pages/**/*.{js,jsx}', + './ui/components/**/*.{js,jsx}', + ], + prefix: "", + theme: { + container: { + center: true, + padding: "2rem", + screens: { + "2xl": "1400px", + }, + }, + extend: { + colors: { + border: "hsl(var(--border))", + input: "hsl(var(--input))", + ring: "hsl(var(--ring))", + background: "hsl(var(--background))", + foreground: "hsl(var(--foreground))", + primary: { + DEFAULT: "hsl(var(--primary))", + foreground: "hsl(var(--primary-foreground))", + }, + secondary: { + DEFAULT: "hsl(var(--secondary))", + foreground: "hsl(var(--secondary-foreground))", + }, + destructive: { + DEFAULT: "hsl(var(--destructive))", + foreground: "hsl(var(--destructive-foreground))", + }, + muted: { + DEFAULT: "hsl(var(--muted))", + foreground: "hsl(var(--muted-foreground))", + }, + accent: { + DEFAULT: "hsl(var(--accent))", + foreground: "hsl(var(--accent-foreground))", + }, + popover: { + DEFAULT: "hsl(var(--popover))", + foreground: "hsl(var(--popover-foreground))", + }, + card: { + DEFAULT: "hsl(var(--card))", + foreground: "hsl(var(--card-foreground))", + }, + }, + borderRadius: { + lg: "var(--radius)", + md: "calc(var(--radius) - 2px)", + sm: "calc(var(--radius) - 4px)", + }, + keyframes: { + "accordion-down": { + from: { height: "0" }, + to: { height: "var(--radix-accordion-content-height)" }, + }, + "accordion-up": { + from: { height: "var(--radix-accordion-content-height)" }, + to: { height: "0" }, + }, + }, + animation: { + "accordion-down": "accordion-down 0.2s ease-out", + "accordion-up": "accordion-up 0.2s ease-out", + }, + }, + }, + plugins: [require("tailwindcss-animate")], +} diff --git a/core/templates/tree/react_express/tsconfig.json b/core/templates/tree/react_express/tsconfig.json new file mode 100644 index 00000000..dc1da568 --- /dev/null +++ b/core/templates/tree/react_express/tsconfig.json @@ -0,0 +1,10 @@ +{ + "compilerOptions": { + "baseUrl": ".", + "paths": { + "@/*": [ + "./ui/*" + ] + } + } +} diff --git a/core/templates/tree/react_express/ui/assets/.gitkeep b/core/templates/tree/react_express/ui/assets/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/core/templates/tree/react_express/ui/components/ui/alert.jsx b/core/templates/tree/react_express/ui/components/ui/alert.jsx new file mode 100644 index 00000000..a9fae7cf --- /dev/null +++ b/core/templates/tree/react_express/ui/components/ui/alert.jsx @@ -0,0 +1,11 @@ +import React from 'react'; +import { AlertCircle } from 'lucide-react'; + +export function AlertDestructive({ title, description }) { + return ( +
+

{title}

+

{description}

+
+ ); +} \ No newline at end of file diff --git a/core/templates/tree/react_express/ui/components/ui/button.jsx b/core/templates/tree/react_express/ui/components/ui/button.jsx new file mode 100644 index 00000000..9bdf65b8 --- /dev/null +++ b/core/templates/tree/react_express/ui/components/ui/button.jsx @@ -0,0 +1,47 @@ +import * as React from "react" +import { Slot } from "@radix-ui/react-slot" +import { cva } from "class-variance-authority"; + +import { cn } from "@/lib/utils" + +const buttonVariants = cva( + "inline-flex items-center justify-center whitespace-nowrap rounded-md text-sm font-medium ring-offset-background transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50", + { + variants: { + variant: { + default: "bg-primary text-primary-foreground hover:bg-primary/90", + destructive: + "bg-destructive text-destructive-foreground hover:bg-destructive/90", + outline: + "border border-input bg-background hover:bg-accent hover:text-accent-foreground", + secondary: + "bg-secondary text-secondary-foreground hover:bg-secondary/80", + ghost: "hover:bg-accent hover:text-accent-foreground", + link: "text-primary underline-offset-4 hover:underline", + }, + size: { + default: "h-10 px-4 py-2", + sm: "h-9 rounded-md px-3", + lg: "h-11 rounded-md px-8", + icon: "h-10 w-10", + }, + }, + defaultVariants: { + variant: "default", + size: "default", + }, + } +) + +const Button = React.forwardRef(({ className, variant, size, asChild = false, ...props }, ref) => { + const Comp = asChild ? Slot : "button" + return ( + () + ); +}) +Button.displayName = "Button" + +export { Button, buttonVariants } diff --git a/core/templates/tree/react_express/ui/components/ui/card.jsx b/core/templates/tree/react_express/ui/components/ui/card.jsx new file mode 100644 index 00000000..dd79b513 --- /dev/null +++ b/core/templates/tree/react_express/ui/components/ui/card.jsx @@ -0,0 +1,50 @@ +import * as React from "react" + +import { cn } from "@/lib/utils" + +const Card = React.forwardRef(({ className, ...props }, ref) => ( +
+)) +Card.displayName = "Card" + +const CardHeader = React.forwardRef(({ className, ...props }, ref) => ( +
+)) +CardHeader.displayName = "CardHeader" + +const CardTitle = React.forwardRef(({ className, ...props }, ref) => ( +

+)) +CardTitle.displayName = "CardTitle" + +const CardDescription = React.forwardRef(({ className, ...props }, ref) => ( +

+)) +CardDescription.displayName = "CardDescription" + +const CardContent = React.forwardRef(({ className, ...props }, ref) => ( +

+)) +CardContent.displayName = "CardContent" + +const CardFooter = React.forwardRef(({ className, ...props }, ref) => ( +
+)) +CardFooter.displayName = "CardFooter" + +export { Card, CardHeader, CardFooter, CardTitle, CardDescription, CardContent } diff --git a/core/templates/tree/react_express/ui/components/ui/input.jsx b/core/templates/tree/react_express/ui/components/ui/input.jsx new file mode 100644 index 00000000..c74d919d --- /dev/null +++ b/core/templates/tree/react_express/ui/components/ui/input.jsx @@ -0,0 +1,19 @@ +import * as React from "react" + +import { cn } from "@/lib/utils" + +const Input = React.forwardRef(({ className, type, ...props }, ref) => { + return ( + () + ); +}) +Input.displayName = "Input" + +export { Input } diff --git a/core/templates/tree/react_express/ui/components/ui/label.jsx b/core/templates/tree/react_express/ui/components/ui/label.jsx new file mode 100644 index 00000000..a1f40999 --- /dev/null +++ b/core/templates/tree/react_express/ui/components/ui/label.jsx @@ -0,0 +1,16 @@ +import * as React from "react" +import * as LabelPrimitive from "@radix-ui/react-label" +import { cva } from "class-variance-authority"; + +import { cn } from "@/lib/utils" + +const labelVariants = cva( + "text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70" +) + +const Label = React.forwardRef(({ className, ...props }, ref) => ( + +)) +Label.displayName = LabelPrimitive.Root.displayName + +export { Label } diff --git a/core/templates/tree/react_express/ui/index.css b/core/templates/tree/react_express/ui/index.css new file mode 100644 index 00000000..42fee7d1 --- /dev/null +++ b/core/templates/tree/react_express/ui/index.css @@ -0,0 +1,82 @@ +@tailwind base; +@tailwind components; +@tailwind utilities; + +@layer base { + :root { + --background: 0 0% 100%; + --foreground: 222.2 47.4% 11.2%; + + --muted: 210 40% 96.1%; + --muted-foreground: 215.4 16.3% 46.9%; + + --popover: 0 0% 100%; + --popover-foreground: 222.2 47.4% 11.2%; + + --border: 214.3 31.8% 91.4%; + --input: 214.3 31.8% 91.4%; + + --card: 0 0% 100%; + --card-foreground: 222.2 47.4% 11.2%; + + --primary: 222.2 47.4% 11.2%; + --primary-foreground: 210 40% 98%; + + --secondary: 210 40% 96.1%; + --secondary-foreground: 222.2 47.4% 11.2%; + + --accent: 210 40% 96.1%; + --accent-foreground: 222.2 47.4% 11.2%; + + --destructive: 0 100% 50%; + --destructive-foreground: 210 40% 98%; + + --ring: 215 20.2% 65.1%; + + --radius: 0.5rem; + } + + .dark { + --background: 224 71% 4%; + --foreground: 213 31% 91%; + + --muted: 223 47% 11%; + --muted-foreground: 215.4 16.3% 56.9%; + + --accent: 216 34% 17%; + --accent-foreground: 210 40% 98%; + + --popover: 224 71% 4%; + --popover-foreground: 215 20.2% 65.1%; + + --border: 216 34% 17%; + --input: 216 34% 17%; + + --card: 224 71% 4%; + --card-foreground: 213 31% 91%; + + --primary: 210 40% 98%; + --primary-foreground: 222.2 47.4% 1.2%; + + --secondary: 222.2 47.4% 11.2%; + --secondary-foreground: 210 40% 98%; + + --destructive: 0 63% 31%; + --destructive-foreground: 210 40% 98%; + + --ring: 216 34% 17%; + + --radius: 0.5rem; + } +} + +@layer base { + * { + @apply border-border; + } + body { + @apply bg-background text-foreground; + font-feature-settings: "rlig" 1, "calt" 1; + } +} + diff --git a/core/templates/tree/react_express/ui/lib/utils.js b/core/templates/tree/react_express/ui/lib/utils.js new file mode 100644 index 00000000..20aa6031 --- /dev/null +++ b/core/templates/tree/react_express/ui/lib/utils.js @@ -0,0 +1,6 @@ +import { clsx } from "clsx" +import { twMerge } from "tailwind-merge" + +export function cn(...inputs) { + return twMerge(clsx(inputs)) +} diff --git a/core/templates/tree/react_express/ui/main.jsx b/core/templates/tree/react_express/ui/main.jsx new file mode 100644 index 00000000..0a5e39af --- /dev/null +++ b/core/templates/tree/react_express/ui/main.jsx @@ -0,0 +1,65 @@ +{% if options.auth %} +import axios from 'axios' +{% endif %} +import React from 'react' +import ReactDOM from 'react-dom/client' +import { createBrowserRouter, RouterProvider, useLocation } from "react-router-dom" + +import './index.css' + +// Pages in the app +import Home from './pages/Home.jsx' +{% if options.auth %} +import Register from './pages/Register.jsx' +import Login from './pages/Login.jsx' + +// Add auth token to every API request if we have it +axios.interceptors.request.use(config => { + const token = localStorage.getItem("token"); + if (token && !config.headers.Authorization) { + config.headers.Authorization = `Token ${token}` + } + return config +}) +{% endif %} + +function PageNotFound() { + const { pathname } = useLocation() + return ( +
+

Page Not Found

+

+ Page {pathname} does not exist. +
+ Go home +

+
+ ); +} + +const router = createBrowserRouter([ + { + path: "/", + element: , + }, +{% if options.auth %} + { + path: "/register/", + element: , + }, + { + path: "/login/", + element: , + }, +{% endif %} + { + path: "*", + element: , + } +]) + +ReactDOM.createRoot(document.getElementById('root')).render( + + + , +) diff --git a/core/templates/tree/react_express/ui/pages/Home.css b/core/templates/tree/react_express/ui/pages/Home.css new file mode 100644 index 00000000..f5b6dc4a --- /dev/null +++ b/core/templates/tree/react_express/ui/pages/Home.css @@ -0,0 +1,6 @@ +#homePage { + max-width: 1280px; + margin: 0 auto; + padding: 2rem; + text-align: center; +} \ No newline at end of file diff --git a/core/templates/tree/react_express/ui/pages/Home.jsx b/core/templates/tree/react_express/ui/pages/Home.jsx new file mode 100644 index 00000000..3f9daa14 --- /dev/null +++ b/core/templates/tree/react_express/ui/pages/Home.jsx @@ -0,0 +1,11 @@ +import './Home.css' + +function Home() { + return ( +
+

{{ project_name }}

+
+ ) +} + +export default Home diff --git a/core/templates/tree/react_express/ui/pages/Login.jsx b/core/templates/tree/react_express/ui/pages/Login.jsx new file mode 100644 index 00000000..0bfffda7 --- /dev/null +++ b/core/templates/tree/react_express/ui/pages/Login.jsx @@ -0,0 +1,94 @@ +import { useState } from "react" +import { useForm } from "react-hook-form" +import { useNavigate } from "react-router-dom" +import { Button } from "@/components/ui/button" +import { Input } from "@/components/ui/input" +import { Label } from "@/components/ui/label" +import { + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from "@/components/ui/card" +import { useToast } from "@/hooks/useToast" +import { LogIn } from "lucide-react" +import { useAuth } from "@/contexts/AuthContext" + +type LoginForm = { + email: string + password: string +} + +export function Login() { + const [loading, setLoading] = useState(false) + const { toast } = useToast() + const navigate = useNavigate() + const { login } = useAuth() + const { register, handleSubmit } = useForm() + + const onSubmit = async (data: LoginForm) => { + try { + setLoading(true) + await login(data.email, data.password) + toast({ + title: "Success", + description: "Logged in successfully", + }) + navigate("/") + } catch (error: any) { + toast({ + variant: "destructive", + title: "Error", + description: error.message || "An error occurred", + }) + } finally { + setLoading(false) + } + } + + return ( +
+ + + Welcome back + Enter your credentials to continue + + +
+
+ + +
+
+ + +
+ +
+ + +
+ ) +} diff --git a/core/templates/tree/react_express/ui/pages/Register.jsx b/core/templates/tree/react_express/ui/pages/Register.jsx new file mode 100644 index 00000000..51e408c7 --- /dev/null +++ b/core/templates/tree/react_express/ui/pages/Register.jsx @@ -0,0 +1,96 @@ +import React, { useState } from 'react'; +import axios from 'axios'; +import { useNavigate } from 'react-router-dom'; +import { AlertDestructive } from "@/components/ui/alert"; +import { Button } from "@/components/ui/button"; +import { + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from "@/components/ui/card"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; + +export default function Register() { + const [email, setEmail] = useState(''); + const [password, setPassword] = useState(''); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(''); + const navigate = useNavigate(); + + const handleSubmit = async (e) => { + e.preventDefault(); + setLoading(true); + setError(''); + try { + const response = await axios.post('/api/auth/register', { email, password }); + if (response.data) { + navigate('/login'); + } + } catch (error) { + console.error('Registration error:', error); + setError(error.response?.data?.error || 'An unexpected error occurred'); + } finally { + setLoading(false); + } + }; + + return ( +
+ + {error && } +
+ + Sign Up + + Enter your information to create an account + + + +
+
+ + setEmail(e.target.value)} + /> +
+
+ + setPassword(e.target.value)} + /> +
+ +
+
+ Already have an account?{" "} + + Sign in + +
+ +
+ +
+ ); +} \ No newline at end of file diff --git a/core/templates/tree/react_express/vite.config.js b/core/templates/tree/react_express/vite.config.js new file mode 100644 index 00000000..80d5d200 --- /dev/null +++ b/core/templates/tree/react_express/vite.config.js @@ -0,0 +1,21 @@ +import { defineConfig } from 'vite' +import react from '@vitejs/plugin-react' +import path from 'path' + +// https://vitejs.dev/config/ +export default defineConfig({ + plugins: [react()], + resolve: { + alias: { + "@": path.resolve(__dirname, "./ui"), + }, + }, + server: { + proxy: { + '/api': { + target: 'http://localhost:3000', + changeOrigin: true, + } + } + } +}) diff --git a/core/templates/tree/vite_react/.gitignore b/core/templates/tree/vite_react/.gitignore new file mode 100644 index 00000000..11cc8717 --- /dev/null +++ b/core/templates/tree/vite_react/.gitignore @@ -0,0 +1,7 @@ +server/node_modules/ +server/.env +server/package-lock.json +client/node_modules/ +client/package-lock.json +node_modules/ +package-lock.json diff --git a/core/templates/tree/vite_react/client/components.json b/core/templates/tree/vite_react/client/components.json new file mode 100644 index 00000000..455581be --- /dev/null +++ b/core/templates/tree/vite_react/client/components.json @@ -0,0 +1,23 @@ +{% raw %} +{ + "$schema": "https://ui.shadcn.com/schema.json", + "style": "default", + "rsc": false, + "tsx": true, + "tailwind": { + "config": "tailwind.config.js", + "css": "src/index.css", + "baseColor": "neutral", + "cssVariables": true, + "prefix": "" + }, + "aliases": { + "components": "@/components", + "utils": "@/lib/utils", + "ui": "@/components/ui", + "lib": "@/lib", + "hooks": "@/hooks" + }, + "iconLibrary": "lucide" +} +{% endraw %} \ No newline at end of file diff --git a/core/templates/tree/vite_react/client/eslint.config.js b/core/templates/tree/vite_react/client/eslint.config.js new file mode 100644 index 00000000..8af0b78f --- /dev/null +++ b/core/templates/tree/vite_react/client/eslint.config.js @@ -0,0 +1,31 @@ +{% raw %} +import js from '@eslint/js' +import globals from 'globals' +import reactHooks from 'eslint-plugin-react-hooks' +import reactRefresh from 'eslint-plugin-react-refresh' +import tseslint from 'typescript-eslint' + +export default tseslint.config( + { ignores: ['dist'] }, + { + extends: [js.configs.recommended, ...tseslint.configs.recommended], + files: ['**/*.{ts,tsx}'], + languageOptions: { + ecmaVersion: 2020, + globals: globals.browser, + }, + plugins: { + 'react-hooks': reactHooks, + 'react-refresh': reactRefresh, + }, + rules: { + ...reactHooks.configs.recommended.rules, + 'react-refresh/only-export-components': [ + 'warn', + { allowConstantExport: true }, + ], + }, + }, +) + +{% endraw %} \ No newline at end of file diff --git a/core/templates/tree/vite_react/client/index.html b/core/templates/tree/vite_react/client/index.html new file mode 100644 index 00000000..927139bb --- /dev/null +++ b/core/templates/tree/vite_react/client/index.html @@ -0,0 +1,13 @@ + + + + + + {{ project_name }} + + + +
+ + + diff --git a/core/templates/tree/vite_react/client/package.json b/core/templates/tree/vite_react/client/package.json new file mode 100644 index 00000000..4104c724 --- /dev/null +++ b/core/templates/tree/vite_react/client/package.json @@ -0,0 +1,81 @@ +{ + "name": "vite_client", + "private": true, + "version": "0.0.0", + "type": "module", + "scripts": { + "start": "npm run dev", + "dev": "vite", + "build": "tsc -b && vite build", + "lint": "eslint .", + "preview": "vite preview" + }, + "dependencies": { + "@hookform/resolvers": "^3.9.1", + "@radix-ui/react-accordion": "^1.2.1", + "@radix-ui/react-alert-dialog": "^1.1.2", + "@radix-ui/react-aspect-ratio": "^1.1.0", + "@radix-ui/react-avatar": "^1.1.1", + "@radix-ui/react-checkbox": "^1.1.2", + "@radix-ui/react-collapsible": "^1.1.1", + "@radix-ui/react-context-menu": "^2.2.2", + "@radix-ui/react-dialog": "^1.1.2", + "@radix-ui/react-dropdown-menu": "^2.1.2", + "@radix-ui/react-hover-card": "^1.1.2", + "@radix-ui/react-label": "^2.1.0", + "@radix-ui/react-menubar": "^1.1.2", + "@radix-ui/react-navigation-menu": "^1.2.1", + "@radix-ui/react-popover": "^1.1.2", + "@radix-ui/react-progress": "^1.1.0", + "@radix-ui/react-radio-group": "^1.2.1", + "@radix-ui/react-scroll-area": "^1.2.1", + "@radix-ui/react-select": "^2.1.2", + "@radix-ui/react-separator": "^1.1.0", + "@radix-ui/react-slider": "^1.2.1", + "@radix-ui/react-slot": "^1.1.0", + "@radix-ui/react-switch": "^1.1.1", + "@radix-ui/react-tabs": "^1.1.1", + "@radix-ui/react-toast": "^1.2.2", + "@radix-ui/react-toggle": "^1.1.0", + "@radix-ui/react-toggle-group": "^1.1.0", + "@radix-ui/react-tooltip": "^1.1.4", + "axios": "^1.7.8", + "class-variance-authority": "^0.7.0", + "clsx": "^2.1.1", + "cmdk": "^1.0.0", + "date-fns": "^3.6.0", + "embla-carousel-react": "^8.5.1", + "input-otp": "^1.4.1", + "lucide-react": "^0.460.0", + "next-themes": "^0.4.3", + "react": "^18.3.1", + "react-day-picker": "^8.10.1", + "react-dom": "^18.3.1", + "react-hook-form": "^7.53.2", + "react-resizable-panels": "^2.1.7", + "react-router-dom": "^7.0.1", + "recharts": "^2.13.3", + "sonner": "^1.7.0", + "tailwind-merge": "^2.5.4", + "tailwindcss-animate": "^1.0.7", + "vaul": "^1.1.1", + "zod": "^3.23.8" + }, + "devDependencies": { + "@eslint/js": "^9.13.0", + "@types/node": "^22.9.1", + "@types/react": "^18.3.12", + "@types/react-dom": "^18.3.1", + "@vitejs/plugin-react": "^4.3.1", + "autoprefixer": "^10.4.20", + "eslint": "^9.13.0", + "eslint-plugin-react-hooks": "^5.0.0", + "eslint-plugin-react-refresh": "^0.4.14", + "globals": "^15.11.0", + "postcss": "^8.4.49", + "tailwindcss": "^3.4.15", + "typescript": "~5.6.2", + "typescript-eslint": "^8.11.0", + "vite": "^5.4.8" + } +} diff --git a/core/templates/tree/vite_react/client/postcss.config.js b/core/templates/tree/vite_react/client/postcss.config.js new file mode 100644 index 00000000..b41d7dd5 --- /dev/null +++ b/core/templates/tree/vite_react/client/postcss.config.js @@ -0,0 +1,9 @@ +{% raw %} +export default { + plugins: { + tailwindcss: {}, + autoprefixer: {}, + }, +} + +{% endraw %} \ No newline at end of file diff --git a/core/templates/tree/vite_react/client/public/favicon.ico b/core/templates/tree/vite_react/client/public/favicon.ico new file mode 100644 index 00000000..0faa111b Binary files /dev/null and b/core/templates/tree/vite_react/client/public/favicon.ico differ diff --git a/core/templates/tree/vite_react/client/src/App.css b/core/templates/tree/vite_react/client/src/App.css new file mode 100644 index 00000000..4865d238 --- /dev/null +++ b/core/templates/tree/vite_react/client/src/App.css @@ -0,0 +1,45 @@ +{% raw %} +#root { + max-width: 1280px; + margin: 0 auto; + padding: 2rem; + text-align: center; +} + +.logo { + height: 6em; + padding: 1.5em; + will-change: filter; + transition: filter 300ms; +} +.logo:hover { + filter: drop-shadow(0 0 2em #646cffaa); +} +.logo.react:hover { + filter: drop-shadow(0 0 2em #61dafbaa); +} + +@keyframes logo-spin { + from { + transform: rotate(0deg); + } + to { + transform: rotate(360deg); + } +} + +@media (prefers-reduced-motion: no-preference) { + a:nth-of-type(2) .logo { + animation: logo-spin infinite 20s linear; + } +} + +.card { + padding: 2em; +} + +.read-the-docs { + color: #888; +} + +{% endraw %} \ No newline at end of file diff --git a/core/templates/tree/vite_react/client/src/App.tsx b/core/templates/tree/vite_react/client/src/App.tsx new file mode 100644 index 00000000..b6cd4f25 --- /dev/null +++ b/core/templates/tree/vite_react/client/src/App.tsx @@ -0,0 +1,26 @@ +import { BrowserRouter as Router, Routes, Route } from "react-router-dom" +import { ThemeProvider } from "./components/ui/theme-provider" +import { Toaster } from "./components/ui/toaster" +import { AuthProvider } from "./contexts/AuthContext" +import { Login } from "./pages/Login" +import { Register } from "./pages/Register" +import { Layout } from "./components/Layout" +import { ProtectedRoute } from "./components/ProtectedRoute" +function App() { + return ( + + + + + } /> + } /> + } /> + + + + + + ) +} + +export default App diff --git a/core/templates/tree/vite_react/client/src/api/api.ts b/core/templates/tree/vite_react/client/src/api/api.ts new file mode 100644 index 00000000..0258aed9 --- /dev/null +++ b/core/templates/tree/vite_react/client/src/api/api.ts @@ -0,0 +1,67 @@ +import axios, { AxiosRequestConfig, AxiosError } from 'axios'; + +const backendURL = ''; +const api = axios.create({ + baseURL: backendURL, + headers: { + 'Content-Type': 'application/json', + }, + validateStatus: (status) => { + return status >= 200 && status < 300; + }, +}); + +let accessToken: string | null = null; +{% if options.auth %} +// Axios request interceptor: Attach access token to headers +api.interceptors.request.use( + (config: AxiosRequestConfig): AxiosRequestConfig => { + if (!accessToken) { + accessToken = localStorage.getItem('accessToken'); + } + if (accessToken && config.headers) { + config.headers.Authorization = `Bearer ${accessToken}`; + } + return config; + }, + (error: AxiosError): Promise => Promise.reject(error) +); + +// Axios response interceptor: Handle 401 errors +api.interceptors.response.use( + (response) => response, // If the response is successful, return it + async (error: AxiosError): Promise => { + const originalRequest = error.config as AxiosRequestConfig & { _retry?: boolean }; + + // If the error is due to an expired access token + if ([401, 403].includes(error.response?.status) && !originalRequest._retry) { + originalRequest._retry = true; // Mark the request as retried + + try { + // Attempt to refresh the token + const { data } = await axios.post<{ accessToken: string }>(`${backendURL}/api/auth/refresh`, { + refreshToken: localStorage.getItem('refreshToken'), + }); + accessToken = data.accessToken; + + // Retry the original request with the new token + if (originalRequest.headers) { + originalRequest.headers.Authorization = `Bearer ${accessToken}`; + } + return api(originalRequest); + } catch (err) { + // If refresh fails, clear tokens and redirect to login + localStorage.removeItem('refreshToken'); + localStorage.removeItem('accessToken'); + accessToken = null; + window.location.href = '/login'; // Redirect to login page + return Promise.reject(err); + } + } + + return Promise.reject(error); // Pass other errors through + } +); +{% endif %} + +export default api; diff --git a/core/templates/tree/vite_react/client/src/api/auth.ts b/core/templates/tree/vite_react/client/src/api/auth.ts new file mode 100644 index 00000000..ea8366d2 --- /dev/null +++ b/core/templates/tree/vite_react/client/src/api/auth.ts @@ -0,0 +1,44 @@ +{% if options.auth %} +import api from './api'; + +// Description: Login user functionality +// Endpoint: POST /api/auth/login +// Request: { email: string, password: string } +// Response: { accessToken: string, refreshToken: string } +export const login = async (email: string, password: string) => { + try { + return { accessToken: '123', refreshToken: '123' }; // this is just a placeholder - remove when the backend is being implemented + const response = await api.post('/api/auth/login', { email, password }); + return response.data; + } catch (error) { + console.error('Login error:', error); + throw new Error(error?.response?.data?.message || error.message); + } +}; + +// Description: Register user functionality +// Endpoint: POST /api/auth/register +// Request: { email: string, password: string } +// Response: { email: string } +export const register = async (email: string, password: string) => { + try { + return { email: 'jake@example.com' }; // this is just a placeholder - remove when the backend is being implemented + const response = await api.post('/api/auth/register', {email, password}); + return response.data; + } catch (error) { + throw new Error(error?.response?.data?.message || error.message); + } +}; + +// Description: Logout +// Endpoint: POST /api/auth/logout +// Request: {} +// Response: { success: boolean, message: string } +export const logout = async () => { + try { + return await api.post('/api/auth/logout'); + } catch (error) { + throw new Error(error?.response?.data?.message || error.message); + } +}; +{% endif %} diff --git a/core/templates/tree/vite_react/client/src/components/Footer.tsx b/core/templates/tree/vite_react/client/src/components/Footer.tsx new file mode 100644 index 00000000..a493341e --- /dev/null +++ b/core/templates/tree/vite_react/client/src/components/Footer.tsx @@ -0,0 +1,13 @@ +{% raw %} +export function Footer() { + return ( + + ) +} +{% endraw %} \ No newline at end of file diff --git a/core/templates/tree/vite_react/client/src/components/Header.tsx b/core/templates/tree/vite_react/client/src/components/Header.tsx new file mode 100644 index 00000000..b104577f --- /dev/null +++ b/core/templates/tree/vite_react/client/src/components/Header.tsx @@ -0,0 +1,35 @@ +{% raw %} +import { Bell{% endraw %}{% if options.auth %}, LogOut{% endif %}{% raw %} } from "lucide-react" +import { Button } from "./ui/button" +import { ThemeToggle } from "./ui/theme-toggle" +import { useAuth } from "@/contexts/AuthContext" +import { useNavigate } from "react-router-dom" + +export function Header() { +{% endraw %} +{% if options.auth %} + const { logout } = useAuth() +{% endif %} + const navigate = useNavigate() +{% if options.auth %} + const handleLogout = () => { + logout() + navigate("/login") + } +{% endif %} + return ( +
+
+
Home
+
+ + {% if options.auth %} + + {% endif %} +
+
+
+ ) +} diff --git a/core/templates/tree/vite_react/client/src/components/Layout.tsx b/core/templates/tree/vite_react/client/src/components/Layout.tsx new file mode 100644 index 00000000..008a7405 --- /dev/null +++ b/core/templates/tree/vite_react/client/src/components/Layout.tsx @@ -0,0 +1,19 @@ +import { Outlet } from "react-router-dom" +import { Header } from "./Header" +import { Footer } from "./Footer" + +export function Layout() { + return ( +
+
+
+
+
+ +
+
+
+
+
+ ) +} diff --git a/core/templates/tree/vite_react/client/src/components/ProtectedRoute.tsx b/core/templates/tree/vite_react/client/src/components/ProtectedRoute.tsx new file mode 100644 index 00000000..8f18cff3 --- /dev/null +++ b/core/templates/tree/vite_react/client/src/components/ProtectedRoute.tsx @@ -0,0 +1,15 @@ +{% raw %} +import { Navigate, useLocation } from "react-router-dom"; +import { useAuth } from "@/contexts/AuthContext"; + +export function ProtectedRoute({ children }: { children: React.ReactNode }) { + const { isAuthenticated } = useAuth(); + const location = useLocation(); + + if (!isAuthenticated) { + return ; + } + + return <>{children}; +} +{% endraw %} diff --git a/core/templates/tree/vite_react/client/src/components/ui/accordion.tsx b/core/templates/tree/vite_react/client/src/components/ui/accordion.tsx new file mode 100644 index 00000000..2df7e37d --- /dev/null +++ b/core/templates/tree/vite_react/client/src/components/ui/accordion.tsx @@ -0,0 +1,59 @@ +{% raw %} +import * as React from "react" +import * as AccordionPrimitive from "@radix-ui/react-accordion" +import { ChevronDown } from "lucide-react" + +import { cn } from "@/lib/utils" + +const Accordion = AccordionPrimitive.Root + +const AccordionItem = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, ...props }, ref) => ( + +)) +AccordionItem.displayName = "AccordionItem" + +const AccordionTrigger = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, children, ...props }, ref) => ( + + svg]:rotate-180", + className + )} + {...props} + > + {children} + + + +)) +AccordionTrigger.displayName = AccordionPrimitive.Trigger.displayName + +const AccordionContent = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, children, ...props }, ref) => ( + +
{children}
+ +)) + +AccordionContent.displayName = AccordionPrimitive.Content.displayName + +export { Accordion, AccordionItem, AccordionTrigger, AccordionContent } + +{% endraw %} \ No newline at end of file diff --git a/core/templates/tree/vite_react/client/src/components/ui/alert-dialog.tsx b/core/templates/tree/vite_react/client/src/components/ui/alert-dialog.tsx new file mode 100644 index 00000000..8ef336c2 --- /dev/null +++ b/core/templates/tree/vite_react/client/src/components/ui/alert-dialog.tsx @@ -0,0 +1,144 @@ +{% raw %} +"use client" + +import * as React from "react" +import * as AlertDialogPrimitive from "@radix-ui/react-alert-dialog" + +import { cn } from "@/lib/utils" +import { buttonVariants } from "@/components/ui/button" + +const AlertDialog = AlertDialogPrimitive.Root + +const AlertDialogTrigger = AlertDialogPrimitive.Trigger + +const AlertDialogPortal = AlertDialogPrimitive.Portal + +const AlertDialogOverlay = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, ...props }, ref) => ( + +)) +AlertDialogOverlay.displayName = AlertDialogPrimitive.Overlay.displayName + +const AlertDialogContent = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, ...props }, ref) => ( + + + + +)) +AlertDialogContent.displayName = AlertDialogPrimitive.Content.displayName + +const AlertDialogHeader = ({ + className, + ...props +}: React.HTMLAttributes) => ( +
+) +AlertDialogHeader.displayName = "AlertDialogHeader" + +const AlertDialogFooter = ({ + className, + ...props +}: React.HTMLAttributes) => ( +
+) +AlertDialogFooter.displayName = "AlertDialogFooter" + +const AlertDialogTitle = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, ...props }, ref) => ( + +)) +AlertDialogTitle.displayName = AlertDialogPrimitive.Title.displayName + +const AlertDialogDescription = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, ...props }, ref) => ( + +)) +AlertDialogDescription.displayName = + AlertDialogPrimitive.Description.displayName + +const AlertDialogAction = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, ...props }, ref) => ( + +)) +AlertDialogAction.displayName = AlertDialogPrimitive.Action.displayName + +const AlertDialogCancel = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, ...props }, ref) => ( + +)) +AlertDialogCancel.displayName = AlertDialogPrimitive.Cancel.displayName + +export { + AlertDialog, + AlertDialogPortal, + AlertDialogOverlay, + AlertDialogTrigger, + AlertDialogContent, + AlertDialogHeader, + AlertDialogFooter, + AlertDialogTitle, + AlertDialogDescription, + AlertDialogAction, + AlertDialogCancel, +} + +{% endraw %} \ No newline at end of file diff --git a/core/templates/tree/vite_react/client/src/components/ui/alert.tsx b/core/templates/tree/vite_react/client/src/components/ui/alert.tsx new file mode 100644 index 00000000..c2152292 --- /dev/null +++ b/core/templates/tree/vite_react/client/src/components/ui/alert.tsx @@ -0,0 +1,62 @@ +{% raw %} +import * as React from "react" +import { cva, type VariantProps } from "class-variance-authority" + +import { cn } from "@/lib/utils" + +const alertVariants = cva( + "relative w-full rounded-lg border p-4 [&>svg~*]:pl-7 [&>svg+div]:translate-y-[-3px] [&>svg]:absolute [&>svg]:left-4 [&>svg]:top-4 [&>svg]:text-foreground", + { + variants: { + variant: { + default: "bg-background text-foreground", + destructive: + "border-destructive/50 text-destructive dark:border-destructive [&>svg]:text-destructive", + }, + }, + defaultVariants: { + variant: "default", + }, + } +) + +const Alert = React.forwardRef< + HTMLDivElement, + React.HTMLAttributes & VariantProps +>(({ className, variant, ...props }, ref) => ( +
+)) +Alert.displayName = "Alert" + +const AlertTitle = React.forwardRef< + HTMLParagraphElement, + React.HTMLAttributes +>(({ className, ...props }, ref) => ( +
+)) +AlertTitle.displayName = "AlertTitle" + +const AlertDescription = React.forwardRef< + HTMLParagraphElement, + React.HTMLAttributes +>(({ className, ...props }, ref) => ( +
+)) +AlertDescription.displayName = "AlertDescription" + +export { Alert, AlertTitle, AlertDescription } + +{% endraw %} \ No newline at end of file diff --git a/core/templates/tree/vite_react/client/src/components/ui/aspect-ratio.tsx b/core/templates/tree/vite_react/client/src/components/ui/aspect-ratio.tsx new file mode 100644 index 00000000..80e9fd0d --- /dev/null +++ b/core/templates/tree/vite_react/client/src/components/ui/aspect-ratio.tsx @@ -0,0 +1,8 @@ +{% raw %} +import * as AspectRatioPrimitive from "@radix-ui/react-aspect-ratio" + +const AspectRatio = AspectRatioPrimitive.Root + +export { AspectRatio } + +{% endraw %} \ No newline at end of file diff --git a/core/templates/tree/vite_react/client/src/components/ui/avatar.tsx b/core/templates/tree/vite_react/client/src/components/ui/avatar.tsx new file mode 100644 index 00000000..242be94c --- /dev/null +++ b/core/templates/tree/vite_react/client/src/components/ui/avatar.tsx @@ -0,0 +1,53 @@ +{% raw %} +"use client" + +import * as React from "react" +import * as AvatarPrimitive from "@radix-ui/react-avatar" + +import { cn } from "@/lib/utils" + +const Avatar = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, ...props }, ref) => ( + +)) +Avatar.displayName = AvatarPrimitive.Root.displayName + +const AvatarImage = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, ...props }, ref) => ( + +)) +AvatarImage.displayName = AvatarPrimitive.Image.displayName + +const AvatarFallback = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, ...props }, ref) => ( + +)) +AvatarFallback.displayName = AvatarPrimitive.Fallback.displayName + +export { Avatar, AvatarImage, AvatarFallback } + +{% endraw %} \ No newline at end of file diff --git a/core/templates/tree/vite_react/client/src/components/ui/badge.tsx b/core/templates/tree/vite_react/client/src/components/ui/badge.tsx new file mode 100644 index 00000000..1bb2e513 --- /dev/null +++ b/core/templates/tree/vite_react/client/src/components/ui/badge.tsx @@ -0,0 +1,39 @@ +{% raw %} +import * as React from "react" +import { cva, type VariantProps } from "class-variance-authority" + +import { cn } from "@/lib/utils" + +const badgeVariants = cva( + "inline-flex items-center rounded-full border px-2.5 py-0.5 text-xs font-semibold transition-colors focus:outline-none focus:ring-2 focus:ring-ring focus:ring-offset-2", + { + variants: { + variant: { + default: + "border-transparent bg-primary text-primary-foreground hover:bg-primary/80", + secondary: + "border-transparent bg-secondary text-secondary-foreground hover:bg-secondary/80", + destructive: + "border-transparent bg-destructive text-destructive-foreground hover:bg-destructive/80", + outline: "text-foreground", + }, + }, + defaultVariants: { + variant: "default", + }, + } +) + +export interface BadgeProps + extends React.HTMLAttributes, + VariantProps {} + +function Badge({ className, variant, ...props }: BadgeProps) { + return ( +
+ ) +} + +export { Badge, badgeVariants } + +{% endraw %} \ No newline at end of file diff --git a/core/templates/tree/vite_react/client/src/components/ui/breadcrumb.tsx b/core/templates/tree/vite_react/client/src/components/ui/breadcrumb.tsx new file mode 100644 index 00000000..909c8bbf --- /dev/null +++ b/core/templates/tree/vite_react/client/src/components/ui/breadcrumb.tsx @@ -0,0 +1,118 @@ +{% raw %} +import * as React from "react" +import { Slot } from "@radix-ui/react-slot" +import { ChevronRight, MoreHorizontal } from "lucide-react" + +import { cn } from "@/lib/utils" + +const Breadcrumb = React.forwardRef< + HTMLElement, + React.ComponentPropsWithoutRef<"nav"> & { + separator?: React.ReactNode + } +>(({ ...props }, ref) =>