Model zoo

.github/workflows/test-eynollah.yml

@@ -24,61 +24,63 @@ jobs:
         sudo rm -rf "$AGENT_TOOLSDIRECTORY"
         df -h
     - uses: actions/checkout@v4
-    - uses: actions/cache/restore@v4
-      id: seg_model_cache
-      with:
-        path: models_layout_v0_5_0
-        key: seg-models
-    - uses: actions/cache/restore@v4
-      id: ocr_model_cache
+
+    - name: Lint with ruff
+      uses: astral-sh/ruff-action@v3
       with:
-        path: models_ocr_v0_5_1
-        key: ocr-models
-    - uses: actions/cache/restore@v4
-      id: bin_model_cache
+        src: "./src"
+
+    - name: Try to restore models_eynollah
+      uses: actions/cache/restore@v4
+      id: all_model_cache
       with:
-        path: default-2021-03-09
-        key: bin-models
+        path: models_eynollah
+        key: models_eynollah
+
     - name: Download models
-      if: steps.seg_model_cache.outputs.cache-hit != 'true' || steps.bin_model_cache.outputs.cache-hit != 'true' || steps.ocr_model_cache.outputs.cache-hit != true
-      run: make models
-    - uses: actions/cache/save@v4
-      if: steps.seg_model_cache.outputs.cache-hit != 'true'
-      with:
-        path: models_layout_v0_5_0
-        key: seg-models
-    - uses: actions/cache/save@v4
-      if: steps.ocr_model_cache.outputs.cache-hit != 'true'
-      with:
-        path: models_ocr_v0_5_1
-        key: ocr-models
+      if: steps.all_model_cache.outputs.cache-hit != 'true'
+      run: |
+        make models
+        ls -la models_eynollah
+
     - uses: actions/cache/save@v4
-      if: steps.bin_model_cache.outputs.cache-hit != 'true'
+      if: steps.all_model_cache.outputs.cache-hit != 'true'
       with:
-        path: default-2021-03-09
-        key: bin-models
+        path: models_eynollah
+        key: models_eynollah
+
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
+
+    # - uses: actions/cache@v4
+    #   with:
+    #     path: |
+    #       path/to/dependencies
+    #       some/other/dependencies
+    #     key: ${{ runner.os }}-${{ hashFiles('**/lockfiles') }}
+
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
         make install-dev EXTRAS=OCR,plotting
         make deps-test EXTRAS=OCR,plotting
-        ls -l models_*
-    - name: Lint with ruff
-      uses: astral-sh/ruff-action@v3
-      with:
-        src: "./src"
+
+    - name: Hard-upgrade torch for debugging
+      run: |
+        python -m pip install --upgrade torch
+
     - name: Test with pytest
       run: make coverage PYTEST_ARGS="-vv --junitxml=pytest.xml"
+
     - name: Get coverage results
       run: |
         coverage report --format=markdown >> $GITHUB_STEP_SUMMARY
         coverage html
         coverage json
         coverage xml
+
     - name: Store coverage results
       uses: actions/upload-artifact@v4
       with:
@@ -88,12 +90,15 @@ jobs:
             pytest.xml
             coverage.xml
             coverage.json
+
     - name: Upload coverage results
       uses: codecov/codecov-action@v4
       with:
         files: coverage.xml
         fail_ci_if_error: false
+
     - name: Test standalone CLI
       run: make smoke-test
+
     - name: Test OCR-D CLI
       run: make ocrd-test

.gitignore

@@ -11,3 +11,4 @@ output.html
 *.tif
 *.sw?
 TAGS
+uv.lock

Makefile

@@ -6,23 +6,17 @@ EXTRAS ?=
 DOCKER_BASE_IMAGE ?= docker.io/ocrd/core-cuda-tf2:latest
 DOCKER_TAG ?= ocrd/eynollah
 DOCKER ?= docker
+WGET = wget -O
 
 #SEG_MODEL := https://qurator-data.de/eynollah/2021-04-25/models_eynollah.tar.gz
 #SEG_MODEL := https://qurator-data.de/eynollah/2022-04-05/models_eynollah_renamed.tar.gz
 # SEG_MODEL := https://qurator-data.de/eynollah/2022-04-05/models_eynollah.tar.gz
 #SEG_MODEL := https://github.com/qurator-spk/eynollah/releases/download/v0.3.0/models_eynollah.tar.gz
 #SEG_MODEL := https://github.com/qurator-spk/eynollah/releases/download/v0.3.1/models_eynollah.tar.gz
-SEG_MODEL := https://zenodo.org/records/17194824/files/models_layout_v0_5_0.tar.gz?download=1
-SEG_MODELFILE = $(notdir $(patsubst %?download=1,%,$(SEG_MODEL)))
-SEG_MODELNAME = $(SEG_MODELFILE:%.tar.gz=%)
-
-BIN_MODEL := https://github.com/qurator-spk/sbb_binarization/releases/download/v0.0.11/saved_model_2021_03_09.zip
-BIN_MODELFILE = $(notdir $(BIN_MODEL))
-BIN_MODELNAME := default-2021-03-09
-
-OCR_MODEL := https://zenodo.org/records/17236998/files/models_ocr_v0_5_1.tar.gz?download=1
-OCR_MODELFILE = $(notdir $(patsubst %?download=1,%,$(OCR_MODEL)))
-OCR_MODELNAME = $(OCR_MODELFILE:%.tar.gz=%)
+#SEG_MODEL := https://zenodo.org/records/17194824/files/models_layout_v0_5_0.tar.gz?download=1
+EYNOLLAH_MODELS_URL := https://zenodo.org/records/17417471/files/models_all_v0_7_0.zip
+EYNOLLAH_MODELS_ZIP = $(notdir $(EYNOLLAH_MODELS_URL))
+EYNOLLAH_MODELS_DIR = $(EYNOLLAH_MODELS_ZIP:%.zip=%)
 
 PYTEST_ARGS ?= -vv --isolate
 
@@ -38,7 +32,7 @@ help:
 	@echo "    install-dev  Install editable with pip"
 	@echo "    deps-test    Install test dependencies with pip"
 	@echo "    models       Download and extract models to $(CURDIR):"
-	@echo "                 $(BIN_MODELNAME) $(SEG_MODELNAME) $(OCR_MODELNAME)"
+	@echo "                 $(EYNOLLAH_MODELS_DIR)"
 	@echo "    smoke-test   Run simple CLI check"
 	@echo "    ocrd-test    Run OCR-D CLI check"
 	@echo "    test         Run unit tests"
@@ -47,34 +41,22 @@ help:
 	@echo "    EXTRAS       comma-separated list of features (like 'OCR,plotting') for 'install' [$(EXTRAS)]"
 	@echo "    DOCKER_TAG   Docker image tag for 'docker' [$(DOCKER_TAG)]"
 	@echo "    PYTEST_ARGS  pytest args for 'test' (Set to '-s' to see log output during test execution, '-vv' to see individual tests. [$(PYTEST_ARGS)]"
-	@echo "    SEG_MODEL    URL of 'models' archive to download for segmentation 'test' [$(SEG_MODEL)]"
-	@echo "    BIN_MODEL    URL of 'models' archive to download for binarization 'test' [$(BIN_MODEL)]"
-	@echo "    OCR_MODEL    URL of 'models' archive to download for binarization 'test' [$(OCR_MODEL)]"
+	@echo "    ALL_MODELS   URL of archive of all models [$(ALL_MODELS)]"
 	@echo ""
 
 # END-EVAL
 
-
-# Download and extract models to $(PWD)/models_layout_v0_5_0
-models: $(BIN_MODELNAME) $(SEG_MODELNAME) $(OCR_MODELNAME)
+# Download and extract models to $(PWD)/models_layout_v0_6_0
+models: $(EYNOLLAH_MODELS_DIR)
 
 # do not download these files if we already have the directories
-.INTERMEDIATE: $(BIN_MODELFILE) $(SEG_MODELFILE) $(OCR_MODELFILE)
-
-$(BIN_MODELFILE):
-	wget -O $@ $(BIN_MODEL)
-$(SEG_MODELFILE):
-	wget -O $@ $(SEG_MODEL)
-$(OCR_MODELFILE):
-	wget -O $@ $(OCR_MODEL)
-
-$(BIN_MODELNAME): $(BIN_MODELFILE)
-	mkdir $@
-	unzip -d $@ $<
-$(SEG_MODELNAME): $(SEG_MODELFILE)
-	tar zxf $<
-$(OCR_MODELNAME): $(OCR_MODELFILE)
-	tar zxf $<
+.INTERMEDIATE: $(EYNOLLAH_MODELS_ZIP)
+
+$(EYNOLLAH_MODELS_ZIP):
+	$(WGET) $@ $(EYNOLLAH_MODELS_URL)
+
+$(EYNOLLAH_MODELS_DIR): $(EYNOLLAH_MODELS_ZIP)
+	unzip $<
 
 build:
 	$(PIP) install build
@@ -88,34 +70,28 @@ install:
 install-dev:
 	$(PIP) install -e .$(and $(EXTRAS),[$(EXTRAS)])
 
-ifeq (OCR,$(findstring OCR, $(EXTRAS)))
-deps-test: $(OCR_MODELNAME)
-endif
-deps-test: $(BIN_MODELNAME) $(SEG_MODELNAME)
+deps-test:
 	$(PIP) install -r requirements-test.txt
-ifeq (OCR,$(findstring OCR, $(EXTRAS)))
-	ln -rs $(OCR_MODELNAME)/* $(SEG_MODELNAME)/
-endif
 
 smoke-test: TMPDIR != mktemp -d
 smoke-test: tests/resources/kant_aufklaerung_1784_0020.tif
 	# layout analysis:
-	eynollah layout -i $< -o $(TMPDIR) -m $(CURDIR)/$(SEG_MODELNAME)
+	eynollah layout -i $< -o $(TMPDIR) -m $(CURDIR)/models_eynollah
 	fgrep -q http://schema.primaresearch.org/PAGE/gts/pagecontent/2019-07-15 $(TMPDIR)/$(basename $(<F)).xml
 	fgrep -c -e TextRegion -e ImageRegion -e SeparatorRegion $(TMPDIR)/$(basename $(<F)).xml
 	# layout, directory mode (skip one, add one):
-	eynollah layout -di $(<D) -o $(TMPDIR) -m $(CURDIR)/$(SEG_MODELNAME)
+	eynollah layout -di $(<D) -o $(TMPDIR) -m $(CURDIR)/models_eynollah
 	test -s $(TMPDIR)/euler_rechenkunst01_1738_0025.xml
 	# mbreorder, directory mode (overwrite):
 	eynollah machine-based-reading-order -di $(<D) -o $(TMPDIR) -m $(CURDIR)/$(SEG_MODELNAME)
 	fgrep -q http://schema.primaresearch.org/PAGE/gts/pagecontent/2019-07-15 $(TMPDIR)/$(basename $(<F)).xml
 	fgrep -c -e RegionRefIndexed $(TMPDIR)/$(basename $(<F)).xml
 	# binarize:
-	eynollah binarization -m $(CURDIR)/$(BIN_MODELNAME) -i $< -o $(TMPDIR)/$(<F)
+	eynollah binarization -m $(CURDIR)/models_eynollah/eynollah-binarization_20210425 -i $< -o $(TMPDIR)/$(<F)
 	test -s $(TMPDIR)/$(<F)
 	@set -x; test "$$(identify -format '%w %h' $<)" = "$$(identify -format '%w %h' $(TMPDIR)/$(<F))"
 	# enhance:
-	eynollah enhancement -m $(CURDIR)/$(SEG_MODELNAME) -sos -i $< -o $(TMPDIR) -O
+	eynollah enhancement -m $(CURDIR)/models_eynollah -sos -i $< -o $(TMPDIR) -O
 	test -s $(TMPDIR)/$(<F)
 	@set -x; test "$$(identify -format '%w %h' $<)" = "$$(identify -format '%w %h' $(TMPDIR)/$(<F))"
 	$(RM) -r $(TMPDIR)
@@ -126,18 +102,16 @@ ocrd-test: tests/resources/kant_aufklaerung_1784_0020.tif
 	cp $< $(TMPDIR)
 	ocrd workspace -d $(TMPDIR) init
 	ocrd workspace -d $(TMPDIR) add -G OCR-D-IMG -g PHYS_0020 -i OCR-D-IMG_0020 $(<F)
-	ocrd-eynollah-segment -w $(TMPDIR) -I OCR-D-IMG -O OCR-D-SEG -P models $(CURDIR)/$(SEG_MODELNAME)
+	ocrd-eynollah-segment -w $(TMPDIR) -I OCR-D-IMG -O OCR-D-SEG -P models $(CURDIR)/models_eynollah
 	result=$$(ocrd workspace -d $(TMPDIR) find -G OCR-D-SEG); \
 	fgrep -q http://schema.primaresearch.org/PAGE/gts/pagecontent/2019-07-15 $(TMPDIR)/$$result && \
 	fgrep -c -e TextRegion -e ImageRegion -e SeparatorRegion $(TMPDIR)/$$result
-	ocrd-sbb-binarize -w $(TMPDIR) -I OCR-D-IMG -O OCR-D-BIN -P model $(CURDIR)/$(BIN_MODELNAME)
-	ocrd-sbb-binarize -w $(TMPDIR) -I OCR-D-SEG -O OCR-D-SEG-BIN -P model $(CURDIR)/$(BIN_MODELNAME) -P operation_level region
+	ocrd-sbb-binarize -w $(TMPDIR) -I OCR-D-IMG -O OCR-D-BIN -P model $(CURDIR)/models_eynollah/eynollah-binarization_20210425 
+	ocrd-sbb-binarize -w $(TMPDIR) -I OCR-D-SEG -O OCR-D-SEG-BIN -P model $(CURDIR)/models_eynollah/eynollah-binarization_20210425  -P operation_level region
 	$(RM) -r $(TMPDIR)
 
 # Run unit tests
-test: export MODELS_LAYOUT=$(CURDIR)/$(SEG_MODELNAME)
-test: export MODELS_OCR=$(CURDIR)/$(OCR_MODELNAME)
-test: export MODELS_BIN=$(CURDIR)/$(BIN_MODELNAME)
+test: export EYNOLLAH_MODELS_DIR := $(CURDIR)
 test:
 	$(PYTHON) -m pytest tests --durations=0 --continue-on-collection-errors $(PYTEST_ARGS)