Skip to content

Commit 59fc26c

Browse files
gburdgburd
authored
Merge pull request #124 from berkeleydb/docs/phase4-final
docs: PDF output + docs CI validation (Phase 4+5 of N)
2 parents d43eeb7 + f8e29bd commit 59fc26c

14 files changed

Lines changed: 1062 additions & 18 deletions

‎.github/workflows/docs.yml‎

Lines changed: 194 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,194 @@
1+
# Documentation validation.
2+
#
3+
# Builds the modernized docs (docs-src/build.py: Markdown -> HTML + man + PDF)
4+
# and runs the validators that lock in the reverse-DocBook migration's
5+
# guarantees. Modeled on ci.yml/fuzz.yml conventions: hard gates block PRs,
6+
# advisory tiers are continue-on-error, and heavy work (PDF/TeX-free but slow,
7+
# external link check) is scheduled-only or best-effort so per-PR jobs stay
8+
# fast.
9+
#
10+
# Tooling comes from the flake dev shell (nix develop) so CI matches local
11+
# exactly -- pandoc, weasyprint, poppler-utils, mandoc, codespell, lychee and
12+
# write-good are all pinned there.
13+
#
14+
# HARD gates (fail the PR): build (HTML+man), no-loss, completeness, spelling,
15+
# internal link-check, man-lint.
16+
# ADVISORY (continue-on-error): prose (write-good), external link-check.
17+
# BEST-EFFORT (scheduled / continue-on-error): PDF build + validation (slow).
18+
19+
name: Docs
20+
21+
on:
22+
push:
23+
branches: [master]
24+
pull_request:
25+
paths:
26+
- 'docs-src/**'
27+
- '.github/workflows/docs.yml'
28+
- 'flake.nix'
29+
- 'dist/RELEASE'
30+
workflow_dispatch:
31+
schedule:
32+
# Weekly (Mon 05:23 UTC): the full run including the slow PDF build and the
33+
# external link check, which are best-effort/skipped on per-PR runs.
34+
- cron: '23 5 * * 1'
35+
36+
concurrency:
37+
group: docs-${{ github.ref }}
38+
cancel-in-progress: true
39+
40+
permissions:
41+
contents: read
42+
43+
jobs:
44+
# ----------------------------------------------------------------------------
45+
# Build HTML + man (fast, always) and run the hard gates that depend on the
46+
# generated output. PDF is built here only on schedule/dispatch (see the
47+
# `pdf` job) so a PR isn't gated on the ~4-minute weasyprint pass.
48+
# ----------------------------------------------------------------------------
49+
build:
50+
name: build + gates (html, man, no-loss, completeness, spelling, links, man-lint)
51+
runs-on: ubuntu-latest
52+
steps:
53+
- uses: actions/checkout@v4
54+
55+
- name: Install Nix (flakes enabled)
56+
uses: cachix/install-nix-action@v27
57+
with:
58+
extra_nix_config: |
59+
experimental-features = nix-command flakes
60+
61+
# 1. BUILD (hard): HTML + man with 0 errors. --no-pdf keeps the PR fast;
62+
# the PDF path is exercised by the `pdf` job (scheduled/best-effort).
63+
- name: Build HTML + man
64+
run: nix develop --command bash -c 'cd docs-src && python3 build.py --no-pdf'
65+
66+
# build.py self-check guards the md->man reshape, PDF book discovery, and
67+
# the .md->.html link rewrite (unit-level, no external tools).
68+
- name: build.py self-check
69+
run: nix develop --command bash -c 'cd docs-src && python3 build.py --selfcheck'
70+
71+
# 2. NO-LOSS GATE (hard): every migrated tree still retains its source
72+
# content (word-multiset retention + no code/section drop). Locks the
73+
# "nothing lost" guarantee against future edits.
74+
- name: No-loss gate (all trees)
75+
run: nix develop --command bash -c 'python3 docs-src/_migrate/verify_all.py'
76+
77+
# 3. COMPLETENESS GATE (hard): every public db.h function is documented
78+
# (28/28), and every uncovered method is on the frozen allowlist -- a
79+
# NEW undocumented API fails.
80+
- name: Completeness gate (API coverage)
81+
run: nix develop --command bash -c 'python3 docs-src/_migrate/man_coverage.py --ci'
82+
83+
# 4. SPELLING (hard on NEW typos): codespell, baselined against the legacy
84+
# typo backlog so only newly-introduced typos fail.
85+
- name: Spelling gate (codespell, baselined)
86+
run: nix develop --command bash -c 'python3 docs-src/_migrate/spellcheck.py'
87+
88+
# 6. INTERNAL LINK CHECK (hard): every link into migrated content must
89+
# resolve. Deferred-tree + un-migrated-asset links are excluded (see
90+
# docs-src/_migrate/lychee.toml). External links are the advisory job.
91+
- name: Internal link check (lychee, offline)
92+
run: |
93+
nix develop --command bash -c \
94+
'shopt -s globstar; lychee --offline --config docs-src/_migrate/lychee.toml --no-progress "docs-build/html/**/*.html"'
95+
96+
# 7. MAN-LINT (hard): 0 ERRORS from mandoc across every generated .3
97+
# (STYLE/WARNING are fine).
98+
- name: Man-lint (mandoc -Tlint, 0 ERRORS)
99+
run: |
100+
nix develop --command bash -c '
101+
e=0
102+
for f in docs-build/man/man3/*.3; do
103+
if mandoc -Tlint "$f" 2>&1 | grep -q "ERROR"; then
104+
echo "ERRORS in $f:"; mandoc -Tlint "$f" 2>&1 | grep "ERROR" | head -3
105+
e=$((e+1))
106+
fi
107+
done
108+
echo "man pages with ERRORs: $e"
109+
test "$e" -eq 0'
110+
111+
# ----------------------------------------------------------------------------
112+
# 5. PROSE (advisory): write-good passive-voice / wordiness counts. This is
113+
# decades-old technical prose -- surface the numbers, never gate.
114+
# ----------------------------------------------------------------------------
115+
prose:
116+
name: prose advisory (write-good)
117+
runs-on: ubuntu-latest
118+
continue-on-error: true
119+
steps:
120+
- uses: actions/checkout@v4
121+
- name: Install Nix (flakes enabled)
122+
uses: cachix/install-nix-action@v27
123+
with:
124+
extra_nix_config: |
125+
experimental-features = nix-command flakes
126+
- name: write-good suggestion counts (per tree)
127+
run: |
128+
nix develop --command bash -c '
129+
shopt -s globstar
130+
total=0
131+
for d in docs-src/api/* docs-src/guides/*; do
132+
[ -d "$d" ] || continue
133+
n=$(write-good "$d"/**/*.md 2>/dev/null | grep -c "on line" || true)
134+
printf "%-40s %5s suggestions\n" "$d" "$n"
135+
total=$((total+n))
136+
done
137+
echo "::notice title=Prose (write-good)::$total advisory suggestion(s) across docs-src"'
138+
139+
# ----------------------------------------------------------------------------
140+
# PDF (best-effort / scheduled): weasyprint renders one PDF per book (~4 min
141+
# over all 13). Not a per-PR gate -- runs on schedule/dispatch, or on a PR
142+
# that touches build.py's PDF path. continue-on-error so a rendering hiccup
143+
# informs without blocking.
144+
# ----------------------------------------------------------------------------
145+
pdf:
146+
name: pdf build + validate (best-effort)
147+
runs-on: ubuntu-latest
148+
continue-on-error: true
149+
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'push'
150+
steps:
151+
- uses: actions/checkout@v4
152+
- name: Install Nix (flakes enabled)
153+
uses: cachix/install-nix-action@v27
154+
with:
155+
extra_nix_config: |
156+
experimental-features = nix-command flakes
157+
- name: Build all outputs (incl. PDF)
158+
run: nix develop --command bash -c 'cd docs-src && timeout 900 python3 build.py'
159+
- name: Validate PDFs (non-empty, page count, title-page version)
160+
run: nix develop --command bash -c 'python3 docs-src/_migrate/validate_pdf.py'
161+
- name: Upload PDFs
162+
if: always()
163+
uses: actions/upload-artifact@v4
164+
with:
165+
name: docs-pdf
166+
path: docs-build/pdf/*.pdf
167+
if-no-files-found: ignore
168+
169+
# ----------------------------------------------------------------------------
170+
# External link check (advisory / scheduled): lychee WITH network, so it
171+
# depends on the reachability of third-party sites. Never gates; scheduled
172+
# and dispatch only so PRs don't wait on the network.
173+
# ----------------------------------------------------------------------------
174+
external-links:
175+
name: external link check (advisory)
176+
runs-on: ubuntu-latest
177+
continue-on-error: true
178+
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
179+
steps:
180+
- uses: actions/checkout@v4
181+
- name: Install Nix (flakes enabled)
182+
uses: cachix/install-nix-action@v27
183+
with:
184+
extra_nix_config: |
185+
experimental-features = nix-command flakes
186+
- name: Build HTML
187+
run: nix develop --command bash -c 'cd docs-src && python3 build.py --no-pdf'
188+
- name: Check external links (network, advisory)
189+
run: |
190+
nix develop --command bash -c '
191+
shopt -s globstar
192+
lychee --no-progress --scheme http --scheme https \
193+
--exclude "localhost" --max-concurrency 8 \
194+
"docs-build/html/**/*.html" || true'

‎docs-src/PLAN.md‎

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -126,12 +126,32 @@ the site (and can attach to GitHub releases).
126126
3. **Man pages** (DONE): `build_man()` -> 787 `*.3` (785 C+STL refentry pages +
127127
`libdb.3` overview + utilities), `mandoc -Tlint` = 0 errors. `man_coverage.py`
128128
reports API coverage (measure-only; the CI gate is phase 5).
129-
4. **PDF** (TODO): pandoc per book (programmer_reference, the GSGs, api_reference)
130-
with a shared LaTeX header. `build.py build_pdf()` is the stubbed seam.
131-
5. **CI** (TODO): `docs.yml` with all validators + the completeness gate (wire
132-
`verify.py` per tree + `man_coverage.py` as hard gates).
129+
4. **PDF** (DONE): `build_pdf()` renders ONE PDF per book (13 books: 2 API refs
130+
+ 9 guides + articles' 2 sub-books) via pandoc(html)->**weasyprint** -- no TeX
131+
toolchain, deterministic, ~3.5 min for all 13. Title page (project + live
132+
version + copyright) + running header/footer via `_templates/pdf-print.css`
133+
(CSS paged-media). Output `docs-build/pdf/<book>.pdf`; `validate_pdf.py`
134+
asserts non-empty + sane page count + version on the title page. Page counts:
135+
api_c 655, programmer_reference 370, api_stl 257, installation 168,
136+
upgrading 164, gsg_txn 119, gsg 96, collections 93, gsg_db_rep 66, bdb-sql 47,
137+
mssgtxt 41, inmemory 20, porting 16.
138+
5. **CI** (DONE): `.github/workflows/docs.yml` (nix devShell for tool parity).
139+
HARD gates: build (html+man, 0 errors), no-loss (`verify_all.py`, all 13
140+
trees), completeness (`man_coverage.py --ci`: 28/28 functions + allowlisted
141+
methods), spelling (`spellcheck.py`, codespell baselined to legacy typos),
142+
internal link-check (`lychee` + `lychee.toml`), man-lint (mandoc, 0 ERRORS).
143+
ADVISORY: prose (write-good). BEST-EFFORT/scheduled: PDF build+validate,
144+
external link-check. The 2 genuinely-undocumented APIs
145+
(`db_env_set_func_assert`, `db_env_set_win_security`) got real stub pages, so
146+
the function gate is a hard 100%.
133147
6. **Publish** (TODO): wire gh-pages to the generated HTML; update the landing
134-
page.
148+
page. Needs: a `pages` job (or step) that runs `build.py`, uploads
149+
`docs-build/html` via `actions/upload-pages-artifact` + `deploy-pages`
150+
(permissions: `pages: write`, `id-token: write`); a top-level `index.md`
151+
linking the 12 book landing pages + the man/PDF outputs; and a decision on
152+
whether PDFs/man are published alongside HTML. The deferred CXX/TCL/java/
153+
csharp trees stay out until regenerated from their native doc tools (their
154+
inbound links are excluded in `lychee.toml`).
135155

136156
### Phase 2 retention (verify.py, mean word retention; 0 hard drops on all)
137157

‎docs-src/_migrate/codespell-baseline.txt‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
docs-src/api/c/db_heap_rid.md indx
2+
docs-src/api/c/db_sql_codegen.md requre
3+
docs-src/api/c/dbcompact.md re-use
4+
docs-src/api/c/dbget.md retun
5+
docs-src/api/c/dbset_flags.md ACI
6+
docs-src/api/c/dbset_partition.md implimented
7+
docs-src/api/c/dbset_partition.md simultaniously
8+
docs-src/api/c/envclose.md unncessary
9+
docs-src/api/c/envget_create_dir.md ponter
10+
docs-src/api/c/envlog_get_config.md ACI
11+
docs-src/api/c/envlog_set_config.md ACI
12+
docs-src/api/c/envset_flags.md ACI
13+
docs-src/api/c/envset_mp_mtxcount.md defualt
14+
docs-src/api/c/envset_thread_id.md re-use
15+
docs-src/api/c/mempfget.md exlusive
16+
docs-src/api/c/mutexget_init.md inital
17+
docs-src/api/c/mutexset_init.md inital
18+
docs-src/api/c/txnbegin.md ACI
19+
docs-src/api/c/txncommit.md ACI
20+
docs-src/api/c/txncommit.md possiblity
21+
docs-src/api/stl/DbstlDbt.md neccessary
22+
docs-src/api/stl/DbstlDbt.md refered
23+
docs-src/api/stl/DbstlElemTraits.md compatiable
24+
docs-src/api/stl/DbstlElemTraits.md contigous
25+
docs-src/api/stl/DbstlElemTraits.md funcitons
26+
docs-src/api/stl/DbstlElemTraits.md singeleton
27+
docs-src/api/stl/ElementHolder.md Wappers
28+
docs-src/api/stl/ElementHolder.md unerlying
29+
docs-src/api/stl/ElementRef.md Wappers
30+
docs-src/api/stl/Element_wrappers.md Wappers
31+
docs-src/api/stl/db_container.md proctected
32+
docs-src/api/stl/db_map.md unequality
33+
docs-src/api/stl/db_multimap.md unequality
34+
docs-src/api/stl/db_vector.md contaienr
35+
docs-src/api/stl/db_vector.md databse
36+
docs-src/api/stl/db_vector_base_iterator.md explictily
37+
docs-src/api/stl/db_vector_iterator.md explictily
38+
docs-src/api/stl/dbstl_global_functions.md exisiting
39+
docs-src/api/stl/dbstl_helper_classes.md Wappers
40+
docs-src/api/stl/stlDbstlDbtoperator_assign.md neccessary
41+
docs-src/api/stl/stlDbstlElemTraitscompare.md compatiable
42+
docs-src/api/stl/stlDbstlElemTraitscompare.md funcitons
43+
docs-src/api/stl/stlDbstlElemTraitscopy.md compatiable
44+
docs-src/api/stl/stlDbstlElemTraitscopy.md funcitons
45+
docs-src/api/stl/stlDbstlElemTraitseof.md compatiable
46+
docs-src/api/stl/stlDbstlElemTraitseof.md funcitons
47+
docs-src/api/stl/stlDbstlElemTraitseq.md compatiable
48+
docs-src/api/stl/stlDbstlElemTraitseq.md funcitons
49+
docs-src/api/stl/stlDbstlElemTraitseq_int_type.md compatiable
50+
docs-src/api/stl/stlDbstlElemTraitseq_int_type.md funcitons
51+
docs-src/api/stl/stlDbstlElemTraitsfind.md compatiable
52+
docs-src/api/stl/stlDbstlElemTraitsfind.md funcitons
53+
docs-src/api/stl/stlDbstlElemTraitsinstance.md singeleton
54+
docs-src/api/stl/stlDbstlElemTraitslength.md compatiable
55+
docs-src/api/stl/stlDbstlElemTraitslength.md funcitons
56+
docs-src/api/stl/stlDbstlElemTraitslt.md compatiable
57+
docs-src/api/stl/stlDbstlElemTraitslt.md funcitons
58+
docs-src/api/stl/stlDbstlElemTraitsmove.md compatiable
59+
docs-src/api/stl/stlDbstlElemTraitsmove.md funcitons
60+
docs-src/api/stl/stlDbstlElemTraitsnot_eof.md compatiable
61+
docs-src/api/stl/stlDbstlElemTraitsnot_eof.md funcitons
62+
docs-src/api/stl/stlDbstlElemTraitsto_char_type.md compatiable
63+
docs-src/api/stl/stlDbstlElemTraitsto_char_type.md funcitons
64+
docs-src/api/stl/stlDbstlElemTraitsto_int_type.md compatiable
65+
docs-src/api/stl/stlDbstlElemTraitsto_int_type.md funcitons
66+
docs-src/api/stl/stlElementHolderoperator__aa.md bahavior
67+
docs-src/api/stl/stlElementHolderoperator__ma.md bahavior
68+
docs-src/api/stl/stlElementHolderoperator_assign.md bahavior
69+
docs-src/api/stl/stlElementHolderoperator_da.md bahavior
70+
docs-src/api/stl/stlElementHolderoperator_decr.md bahavior
71+
docs-src/api/stl/stlElementHolderoperator_gt_ge.md bahavior
72+
docs-src/api/stl/stlElementHolderoperator_ia.md bahavior
73+
docs-src/api/stl/stlElementHolderoperator_incr.md bahavior
74+
docs-src/api/stl/stlElementHolderoperator_lt_le.md bahavior
75+
docs-src/api/stl/stlElementHolderoperator_modasg.md bahavior
76+
docs-src/api/stl/stlElementHolderoperator_oa.md bahavior
77+
docs-src/api/stl/stlElementHolderoperator_sa.md bahavior
78+
docs-src/api/stl/stlElementHolderoperator_xa.md bahavior
79+
docs-src/api/stl/stlElementRefElementRef.md unerlying
80+
docs-src/api/stl/stldb_mapinsert.md similiar
81+
docs-src/api/stl/stldb_mapoperator_ueq.md unequality
82+
docs-src/api/stl/stldb_multimapoperator_ueq.md unequality
83+
docs-src/api/stl/stldb_vector_base_iteratordstr_db_vector_base_iterator.md explictily
84+
docs-src/api/stl/stldb_vector_base_iteratoroperator_sub.md substract
85+
docs-src/api/stl/stldb_vector_iteratordstr_db_vector_iterator.md explictily
86+
docs-src/api/stl/stldb_vector_iteratoroperator_sub.md substract
87+
docs-src/api/stl/stldb_vectorassign.md requirs
88+
docs-src/api/stl/stldb_vectorunique.md dertermine
89+
docs-src/api/stl/stldbstl_global_functionscommit_txn.md funcion
90+
docs-src/api/stl/stldbstl_global_functionsset_current_txn_handle.md commiting
91+
docs-src/api/stl/stldbstl_global_functionsset_global_dbfile_suffix_number.md exisiting
92+
docs-src/guides/articles/inmemory/index.md desireable
93+
docs-src/guides/articles/mssgtxt/index.md commiting
94+
docs-src/guides/articles/mssgtxt/index.md compresssion
95+
docs-src/guides/articles/mssgtxt/index.md connnect
96+
docs-src/guides/articles/mssgtxt/index.md indx
97+
docs-src/guides/articles/mssgtxt/index.md paritions
98+
docs-src/guides/articles/mssgtxt/index.md spcified
99+
docs-src/guides/articles/mssgtxt/index.md traget
100+
docs-src/guides/bdb-sql/sqlrep.md operatons
101+
docs-src/guides/gsg_db_rep/elections.md desireable
102+
docs-src/guides/gsg_db_rep/fwrkmasterreplica.md applicaton
103+
docs-src/guides/gsg_db_rep/rep_init_code.md peformed
104+
docs-src/guides/installation/build_android_jdbc.md exisits
105+
docs-src/guides/installation/build_unix_conf.md Documenation
106+
docs-src/guides/installation/changelog_4_8.md Millenium
107+
docs-src/guides/installation/changelog_4_8.md hearbeat
108+
docs-src/guides/installation/changelog_4_8.md partically
109+
docs-src/guides/installation/changelog_4_8.md redefinitons
110+
docs-src/guides/installation/changelog_5_0.md Millenium
111+
docs-src/guides/installation/changelog_5_0.md datbase
112+
docs-src/guides/installation/changelog_5_0.md eqivalent
113+
docs-src/guides/installation/changelog_5_0.md mulitple
114+
docs-src/guides/installation/changelog_5_0.md resouces
115+
docs-src/guides/installation/changelog_5_0.md segementation
116+
docs-src/guides/installation/changelog_5_0.md simulatenously
117+
docs-src/guides/installation/changelog_5_0.md teh
118+
docs-src/guides/installation/changelog_5_0.md unitialized
119+
docs-src/guides/installation/changelog_5_1.md datbase
120+
docs-src/guides/installation/changelog_5_1.md explict
121+
docs-src/guides/installation/changelog_5_1.md numer
122+
docs-src/guides/installation/changelog_5_1.md operaton
123+
docs-src/guides/installation/changelog_5_1.md segementation
124+
docs-src/guides/installation/changelog_5_2.md re-use
125+
docs-src/guides/installation/changelog_5_3.md Enhaced
126+
docs-src/guides/installation/changelog_5_3.md begining
127+
docs-src/guides/installation/changelog_5_3.md dependant
128+
docs-src/guides/installation/introduction.md infomation
129+
docs-src/guides/installation/upgrade_11gr2_52_repmgr_channels.md asychronous
130+
docs-src/guides/installation/upgrade_11gr2_52_repmgr_channels.md sychronous
131+
docs-src/guides/installation/upgrade_11gr2_52_xa.md Applictions
132+
docs-src/guides/installation/upgrade_11gr2_remsupp.md Millenium
133+
docs-src/guides/porting/certport.md thrid
134+
docs-src/guides/porting/certport.md warninigs
135+
docs-src/guides/porting/modscope.md envrionment
136+
docs-src/guides/porting/modscope.md platfrom
137+
docs-src/guides/programmer_reference/arch_apis.md extention
138+
docs-src/guides/programmer_reference/bt_conf.md preceeding
139+
docs-src/guides/programmer_reference/ch13s02.md pre-emptive
140+
docs-src/guides/programmer_reference/csharp.md libaries
141+
docs-src/guides/programmer_reference/embedded.md seemlessly
142+
docs-src/guides/programmer_reference/env_encrypt.md Documenation
143+
docs-src/guides/programmer_reference/intro_products.md informaion
144+
docs-src/guides/programmer_reference/lock_max.md enviroment
145+
docs-src/guides/programmer_reference/mp_warm.md intialize
146+
docs-src/guides/programmer_reference/program_perfmon.md stap
147+
docs-src/guides/programmer_reference/program_ram.md re-used
148+
docs-src/guides/programmer_reference/stl_examples.md squre
149+
docs-src/guides/programmer_reference/stl_usecase.md prefered
150+
docs-src/guides/programmer_reference/transapp_atomicity.md ACI
151+
docs-src/guides/programmer_reference/transapp_throughput.md ACI
152+
docs-src/guides/programmer_reference/txn_config.md ACI
153+
docs-src/guides/upgrading/changelog_4_7.md invalide

0 commit comments

Comments
 (0)