Schemas 2.0

.github/workflows/black.yml

@@ -6,6 +6,6 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - uses: psf/black@stable
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+      - uses: psf/black@stable

.github/workflows/deploy-api-dev.yml

@@ -0,0 +1,78 @@
+# This workflow will build and push a new container image to Amazon ECR,
+# and then will deploy a new task definition to Amazon ECS, when a release is created
+#
+# To use this workflow, you will need to complete the following set-up steps:
+#
+# 1. Create an ECR repository to store your images.
+#    For example: `aws ecr create-repository --repository-name my-ecr-repo --region us-east-2`.
+#    Replace the value of `ECR_REPOSITORY` in the workflow below with your repository's name.
+#    Replace the value of `aws-region` in the workflow below with your repository's region.
+#
+# 2. Create an ECS task definition, an ECS cluster, and an ECS service.
+#    For example, follow the Getting Started guide on the ECS console:
+#      https://us-east-2.console.aws.amazon.com/ecs/home?region=us-east-2#/firstRun
+#    Replace the values for `service` and `cluster` in the workflow below with your service and cluster names.
+#
+# 3. Store your ECS task definition as a JSON file in your repository.
+#    The format should follow the output of `aws ecs register-task-definition --generate-cli-skeleton`.
+#    Replace the value of `task-definition` in the workflow below with your JSON file's name.
+#    Replace the value of `container-name` in the workflow below with the name of the container
+#    in the `containerDefinitions` section of the task definition.
+#
+# 4. Store an IAM user access key in GitHub Actions secrets named `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`.
+#    See the documentation for each action used below for the recommended IAM policies for this IAM user,
+#    and best practices on handling the access key credentials.
+
+on:
+  workflow_dispatch:
+    inputs: null
+
+name: Deploy API to AWS - dev
+
+jobs:
+  deploy:
+    env:
+      server: dev
+    name: Deploy
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-1
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v1
+
+      - name: Build, tag, and push image to Amazon ECR
+        id: build-image
+        env:
+          REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          REPOSITORY: pephub_dev
+          IMAGE_TAG: ${{ github.sha }}
+        run: |
+          docker build -t $REGISTRY/$REPOSITORY:$IMAGE_TAG -f deployment/pephub.databio.org/Dockerfile_dev .
+          docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG
+          echo "image=$REGISTRY/$REPOSITORY:$IMAGE_TAG" >> $GITHUB_OUTPUT
+
+      - name: Fill in the new image ID in the Amazon ECS task definition
+        id: task-def
+        uses: aws-actions/amazon-ecs-render-task-definition@v1
+        with:
+          task-definition: deployment/task_defs/dev.json
+          container-name: pephub-container
+          image: ${{ steps.build-image.outputs.image }}
+
+      - name: Deploy Amazon ECS task definition
+        uses: aws-actions/amazon-ecs-deploy-task-definition@v1
+        with:
+          task-definition: ${{ steps.task-def.outputs.task-definition }}
+          service: pephub-service-dev
+          cluster: yeti
+          wait-for-service-stability: true

.github/workflows/deploy-api.yml

@@ -24,9 +24,6 @@
 #    and best practices on handling the access key credentials.
 
 on:
-  push:
-    branches:
-      - master
   workflow_dispatch:
     inputs: null
 
@@ -60,7 +57,7 @@ jobs:
           REPOSITORY: pephub
           IMAGE_TAG: ${{ github.sha }}
         run: |
-          docker build -t $REGISTRY/$REPOSITORY:$IMAGE_TAG -f deployment/pephub.databio.org/Dockerfile .
+          docker build -t $REGISTRY/$REPOSITORY:$IMAGE_TAG -f deployment/pephub.databio.org/Dockerfile_primary .
           docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG
           echo "image=$REGISTRY/$REPOSITORY:$IMAGE_TAG" >> $GITHUB_OUTPUT
 

.github/workflows/deploy-dockerhub-dev.yml

@@ -0,0 +1,29 @@
+name: Build and push dev image to Dockerhub
+
+on:
+  workflow_dispatch:
+    inputs: null
+
+jobs:
+  deploy:
+    name: Deploy
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Build and Push to DockerHub
+        uses: docker/build-push-action@v5
+        with:
+          push: true
+          file: deployment/dockerhub/dev.Dockerfile
+          tags: "databio/pephub:dev"

.github/workflows/deploy-dockerhub.yml

@@ -1,12 +1,11 @@
+name: Deploy to Dockerhub on release
+
 on:
   release:
-    types:
-      - created
+    types: [published]
   workflow_dispatch:
     inputs: null
 
-name: Deploy to Dockerhub on release
-
 jobs:
   deploy:
     name: Deploy
@@ -28,5 +27,5 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           push: true
-          file: deployment/dockerhub/Dockerfile
+          file: deployment/dockerhub/primary.Dockerfile
           tags: databio/pephub:latest, databio/pephub:${{ github.ref_name }}

deployment/dockerhub/dev.Dockerfile

@@ -0,0 +1,22 @@
+# -------------
+# BUILD BACKEND
+# -------------
+FROM python:3.10-slim
+LABEL authors="Nathan LeRoy, Nathan Sheffield, Oleksandr Khoroshevskyi"
+
+RUN apt-get update
+RUN apt-get install -y gcc
+RUN apt-get install -y libpq-dev
+RUN apt-get install -y --no-install-recommends git
+
+EXPOSE 5432
+EXPOSE 6333
+EXPOSE 80
+
+WORKDIR /app
+COPY . /app
+
+RUN python -m pip install --upgrade pip
+RUN pip install -r requirements/requirements-all.txt --no-cache-dir
+
+CMD ["uvicorn", "pephub.main:app", "--host", "0.0.0.0", "--port", "80"]

deployment/pephub.databio.org/Dockerfile_dev

@@ -0,0 +1,2 @@
+FROM databio/pephub:dev
+# pass - nothing else is needed

deployment/task_defs/dev.json

@@ -0,0 +1,116 @@
+{
+  "ipcMode": null,
+  "executionRoleArn": "arn:aws:iam::235728444054:role/ecsTaskExecutionRole",
+  "cpu": null,
+  "memory": null,
+  "family": "pephub-task",
+  "containerDefinitions": [
+    {
+      "name": "pephub-container",
+      "cpu": 32,
+      "memory": 6096,
+      "memoryReservation": 512,
+      "portMappings": [
+        {
+          "hostPort": 8121,
+          "protocol": "tcp",
+          "containerPort": 80
+        }
+      ],
+      "secrets": [
+        {
+          "valueFrom": "POSTGRES_HOST_AWS",
+          "name": "POSTGRES_HOST"
+        },
+        {
+          "valueFrom": "PEPHUB_POSTGRES_DB_DEV",
+          "name": "POSTGRES_DB"
+        },
+        {
+          "valueFrom": "PEPHUB_POSTGRES_PASSWORD",
+          "name": "POSTGRES_PASSWORD"
+        },
+        {
+          "valueFrom": "PEPHUB_POSTGRES_USER",
+          "name": "POSTGRES_USER"
+        },
+        {
+          "valueFrom": "PEPHUB_GH_CLIENT_ID",
+          "name": "GH_CLIENT_ID"
+        },
+        {
+          "valueFrom": "PEPHUB_GH_CLIENT_SECRET",
+          "name": "GH_CLIENT_SECRET"
+        },
+        {
+          "valueFrom": "PEPHUB_GH_REDIRECT_URI",
+          "name": "REDIRECT_URI"
+        },
+        {
+          "valueFrom": "PEPHUB_BASE_URI",
+          "name": "BASE_URI"
+        },
+        {
+          "valueFrom": "QDRANT_HOST",
+          "name": "QDRANT_HOST"
+        },
+        {
+          "valueFrom": "QDRANT_PORT",
+          "name": "QDRANT_PORT"
+        },
+        {
+          "valueFrom": "QDRANT_ENABLED",
+          "name": "QDRANT_ENABLED"
+        },
+        {
+          "valueFrom": "QDRANT_COLLECTION",
+          "name": "QDRANT_COLLECTION"
+        },
+        {
+          "valueFrom": "QDRANT_API_KEY",
+          "name": "QDRANT_API_KEY"
+        }
+      ],
+      "dnsSearchDomains": null,
+      "environmentFiles": null,
+      "logConfiguration": null,
+      "entryPoint": null,
+      "command": null,
+      "linuxParameters": null,
+      "environment": null,
+      "resourceRequirements": null,
+      "ulimits": null,
+      "dnsServers": null,
+      "mountPoints": [],
+      "workingDirectory": null,
+      "dockerSecurityOptions": null,
+      "volumesFrom": [],
+      "stopTimeout": null,
+      "image": "235728444054.dkr.ecr.us-east-1.amazonaws.com/my-ecr-repo:170afd5cf39d9799e926e1d0ebf40b9051fb731f",
+      "startTimeout": null,
+      "firelensConfiguration": null,
+      "dependsOn": null,
+      "disableNetworking": null,
+      "interactive": null,
+      "healthCheck": null,
+      "essential": true,
+      "links": null,
+      "hostname": null,
+      "extraHosts": null,
+      "pseudoTerminal": null,
+      "user": null,
+      "readonlyRootFilesystem": null,
+      "dockerLabels": null,
+      "systemControls": null,
+      "privileged": null
+    }
+  ],
+  "placementConstraints": [],
+  "taskRoleArn": "ecsTaskExecutionRole",
+  "pidMode": null,
+  "requiresCompatibilities": ["EC2"],
+  "networkMode": "bridge",
+  "inferenceAccelerators": null,
+  "proxyConfiguration": null,
+  "volumes": []
+}

launch_docker.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-docker run -p 80:80 \
+docker run -p 8000:80 \
     --env POSTGRES_HOST=$POSTGRES_HOST \
     --env POSTGRES_DB=$POSTGRES_DB \
     --env POSTGRES_USER=$POSTGRES_USER \
@@ -9,8 +9,7 @@ docker run -p 80:80 \
     --env QDRANT_PORT=$QDRANT_PORT \
     --env QDRANT_ENABLED=$QDRANT_ENABLED \
     --env QDRANT_API_KEY=$QDRANT_API_KEY \
-    --env HF_MODEL=$HF_MODEL \
     --env GH_CLIENT_ID=$GH_CLIENT_ID \
     --env GH_CLIENT_SECRET=$GH_CLIENT_SECRET \
     --env BASE_URI=$BASE_URI \
-    pephub
+    databio/pephub:dev

pephub/_version.py

@@ -1 +1 @@
-__version__ = "0.14.3"
+__version__ = "0.15.0_dev0"

pephub/dependencies.py

@@ -157,7 +157,7 @@ def read_authorization_header(authorization: str = Header(None)) -> Union[dict,
 
 
 def get_organizations_from_session_info(
-    session_info: Union[dict, None] = Depends(read_authorization_header)
+    session_info: Union[dict, None] = Depends(read_authorization_header),
 ) -> List[str]:
     organizations = []
     if session_info:
@@ -168,7 +168,7 @@ def get_organizations_from_session_info(
 
 
 def get_user_from_session_info(
-    session_info: Union[dict, None] = Depends(read_authorization_header)
+    session_info: Union[dict, None] = Depends(read_authorization_header),
 ) -> Union[str, None]:
     user = None
     if session_info:
@@ -405,8 +405,16 @@ def get_namespace_info(
 
 
 @cached(TTLCache(maxsize=100, ttl=5 * 60))
-def get_pepdb_namespace_info(limit: int = 10) -> ListOfNamespaceInfo:
+def get_pepdb_namespace_info(
+    page: int = 0,
+    page_size: int = 10,
+    order_by: str = "number_of_projects",
+) -> ListOfNamespaceInfo:
     """
     Get the information on the biggest namespaces in the database.
     """
-    return agent.namespace.info(limit=limit)
+    return agent.namespace.info(
+        page=page,
+        page_size=page_size,
+        order_by=order_by,
+    )