-
-
Notifications
You must be signed in to change notification settings - Fork 144
Expand file tree
/
Copy pathjustfile
More file actions
221 lines (197 loc) · 8.45 KB
/
Copy pathjustfile
File metadata and controls
221 lines (197 loc) · 8.45 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
HAS_GPU := `command -v nvidia-smi > /dev/null && echo 1 || echo 0`
# Black's output changes between releases, so a stale local venv can pass
# `just check_format` while CI (which installs dev-requirements.txt) fails.
# The formatter targets below assert the installed version matches this pin.
BLACK_PIN := `sed -n 's/^black==//p' dev-requirements.txt`
help:
@echo "PDF Document Layout Analysis - Available Commands:"
@echo ""
@echo "📄 PDF Analysis with Translation (API + Gradio UI + Ollama):"
@echo " just start - Auto-detects GPU, starts API, UI and translation"
@echo " just start_no_gpu - Forces CPU mode, starts API, UI and translation"
@echo " just start_detached - Background mode, API only (CPU)"
@echo " just start_detached_gpu - Background mode, API only (GPU)"
@echo ""
@echo "📄 Without Translation (API + Gradio UI only):"
@echo " just start_no_translation - Auto-detects GPU, no Ollama"
@echo " just start_no_translation_no_gpu - Forces CPU mode, no Ollama"
@echo ""
@echo "🧪 Testing & Utilities:"
@echo " just test - Run Python tests"
@echo " just stop - Stop all services"
@echo ""
@echo "🔧 Development:"
@echo " just install_venv - Create virtual environment"
@echo " just install - Install dependencies"
@echo " just formatter - Format code with black"
@echo " just check_format - Check code formatting"
@echo ""
@echo "🧹 Cleanup:"
@echo " just remove_docker_containers - Remove Docker containers"
@echo " just remove_docker_images - Remove Docker images"
@echo " just free_up_space - Free up system space"
@echo ""
@echo "💡 Tip: 'just start' launches API (port 5060), UI (port 7860) and translation support"
@echo "💡 GPU stack override: GPU_STACK_PROFILE=legacy|nextgen|grace just start"
install:
. .venv/bin/activate; pip install -Ur requirements.txt
activate:
. .venv/bin/activate
install_venv:
python -m venv .venv
. .venv/bin/activate; python -m pip install --upgrade pip
. .venv/bin/activate; python -m pip install -r dev-requirements.txt
check_black_version:
#!/usr/bin/env bash
set -euo pipefail
. .venv/bin/activate
installed="$( { command black --version 2>/dev/null || true; } | sed -n 's/^black, \([^ ]*\).*/\1/p')"
if [ "$installed" != "{{BLACK_PIN}}" ]; then
echo "black $installed does not match dev-requirements.txt pin {{BLACK_PIN}}" >&2
echo "Run 'just install_venv' to install the pinned version." >&2
exit 1
fi
formatter: check_black_version
. .venv/bin/activate; command black .
check_format: check_black_version
. .venv/bin/activate; command black . --check
remove_docker_containers:
docker compose ps -q | xargs docker rm
remove_docker_images:
docker compose config --images | xargs docker rmi
start:
#!/bin/bash
mkdir -p ./models
if [ {{HAS_GPU}} -eq 1 ]; then
echo "NVIDIA GPU detected, starting with translation support (GPU-enabled Ollama)"
bash select_gpu_stack.sh .docker.gpu.env
echo "Starting Ollama GPU container first..."
docker compose --env-file .docker.gpu.env -f docker-compose-gpu.yml up -d ollama-gpu
echo "Waiting for Ollama to be healthy..."
timeout=60
while [ $timeout -gt 0 ]; do
if docker inspect --format='{{"{{"}}.State.Health.Status{{"}}"}}' ollama-service-gpu 2>/dev/null | grep -q "healthy"; then
echo "Ollama GPU container is healthy!"
break
fi
echo "Waiting for Ollama GPU container to be healthy... ($timeout seconds remaining)"
sleep 5
timeout=$((timeout-5))
done
if ! docker inspect --format='{{"{{"}}.State.Health.Status{{"}}"}}' ollama-service-gpu 2>/dev/null | grep -q "healthy"; then
echo "Warning: Ollama GPU container may not be fully healthy yet, but continuing..."
fi
echo "Starting all services with translation support..."
docker compose --env-file .docker.gpu.env -f docker-compose-gpu.yml up --build pdf-document-layout-analysis-gpu pdf-document-layout-analysis-gui-gpu
else
echo "No NVIDIA GPU detected, starting with translation support (CPU Ollama)"
echo "Starting Ollama container first..."
docker compose -f docker-compose.yml up -d ollama
echo "Waiting for Ollama to be healthy..."
timeout=60
while [ $timeout -gt 0 ]; do
if docker inspect --format='{{"{{"}}.State.Health.Status{{"}}"}}' ollama-service 2>/dev/null | grep -q "healthy"; then
echo "Ollama container is healthy!"
break
fi
echo "Waiting for Ollama container to be healthy... ($timeout seconds remaining)"
sleep 5
timeout=$((timeout-5))
done
if ! docker inspect --format='{{"{{"}}.State.Health.Status{{"}}"}}' ollama-service 2>/dev/null | grep -q "healthy"; then
echo "Warning: Ollama container may not be fully healthy yet, but continuing..."
fi
echo "Starting all services with translation support..."
docker compose -f docker-compose.yml up --build pdf-document-layout-analysis pdf-document-layout-analysis-gui
fi
start_no_gpu:
#!/bin/bash
mkdir -p ./models
echo "Starting with CPU-only configuration and translation support"
echo "Starting Ollama container first..."
docker compose up -d ollama
echo "Waiting for Ollama to be healthy..."
timeout=60
while [ $timeout -gt 0 ]; do
if docker inspect --format='{{"{{"}}.State.Health.Status{{"}}"}}' ollama-service 2>/dev/null | grep -q "healthy"; then
echo "Ollama container is healthy!"
break
fi
echo "Waiting for Ollama container to be healthy... ($timeout seconds remaining)"
sleep 5
timeout=$((timeout-5))
done
if ! docker inspect --format='{{"{{"}}.State.Health.Status{{"}}"}}' ollama-service 2>/dev/null | grep -q "healthy"; then
echo "Warning: Ollama container may not be fully healthy yet, but continuing..."
fi
echo "Starting all services with translation support..."
docker compose up --build pdf-document-layout-analysis pdf-document-layout-analysis-gui
start_no_translation:
mkdir -p ./models
if [ {{HAS_GPU}} -eq 1 ]; then \
echo "NVIDIA GPU detected, using docker-compose-gpu.yml"; \
bash select_gpu_stack.sh .docker.gpu.env; \
docker compose --env-file .docker.gpu.env -f docker-compose-gpu.yml up --build pdf-document-layout-analysis-gpu pdf-document-layout-analysis-gui-gpu; \
else \
echo "No NVIDIA GPU detected, using docker-compose.yml"; \
docker compose -f docker-compose.yml up --build pdf-document-layout-analysis pdf-document-layout-analysis-gui; \
fi
start_no_translation_no_gpu:
mkdir -p ./models
@echo "Starting with CPU-only configuration (no translation support)"
docker compose up --build pdf-document-layout-analysis pdf-document-layout-analysis-gui
stop:
docker compose stop
docker compose -f docker-compose-gpu.yml stop
test:
. .venv/bin/activate; command cd src; command python -m pytest
free_up_space:
df -h
sudo rm -rf /usr/share/dotnet
sudo rm -rf /opt/ghc
sudo rm -rf "/usr/local/share/boost"
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
sudo apt-get remove -y '^llvm-.*' || true
sudo apt-get remove -y 'php.*' || true
sudo apt-get remove -y google-cloud-sdk hhvm google-chrome-stable firefox mono-devel || true
sudo apt-get autoremove -y
sudo apt-get clean
sudo rm -rf /usr/share/dotnet
sudo rm -rf /usr/local/lib/android
sudo rm -rf /opt/hostedtoolcache/CodeQL
sudo docker image prune --all --force
df -h
start_detached:
mkdir -p ./models
@echo "Starting in detached mode"
docker compose up --build -d pdf-document-layout-analysis
@echo "Main application started in background. Check status with: docker compose ps"
@echo "View logs with: docker compose logs -f pdf-document-layout-analysis"
start_detached_gpu:
mkdir -p ./models
@echo "Starting in detached mode with GPU"
bash select_gpu_stack.sh .docker.gpu.env
RESTART_IF_NO_GPU=true docker compose --env-file .docker.gpu.env -f docker-compose-gpu.yml up --build -d pdf-document-layout-analysis-gpu
@echo "Main application started in background. Check status with: docker compose ps"
@echo "View logs with: docker compose logs -f pdf-document-layout-analysis-gpu"
upgrade:
. .venv/bin/activate; pip-upgrade
tag:
#!/bin/bash
# Get current date
CURRENT_DATE=$(date +%Y.%-m.%-d)
echo "Current date: $CURRENT_DATE"
# Get the latest tag that matches today's date pattern
LATEST_TAG=$(git tag --list "${CURRENT_DATE}.*" --sort=-version:refname | head -n1)
if [ -z "$LATEST_TAG" ]; then
# No tag for today, start with revision 1
REVISION=1
else
# Extract revision number and increment
REVISION=$(echo $LATEST_TAG | cut -d. -f4)
REVISION=$((REVISION + 1))
fi
NEW_TAG="${CURRENT_DATE}.${REVISION}"
echo "Creating new tag: $NEW_TAG"
git tag $NEW_TAG
git push --tag