Merge branch 'main' into PG-1013-DOC-pg_tde-for-PG17

.github/workflows/docker.yaml

@@ -17,7 +17,7 @@ jobs:
 
     steps:
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1
+        uses: docker/setup-buildx-action@6524bf65af31da8d45b59e8c27de4bd072b392f5 # v3.8.0
 
       - name: Build
         uses: docker/build-push-action@48aba3b46d1b1fec4febb7c5d0c644b249a11355 # v6.10.0

.github/workflows/docs.yaml

@@ -1,13 +1,14 @@
 name: Docs
 on:
+  workflow_dispatch: {}
   push:
     branches:
       - main
     paths:
       - "documentation/**"
 
 permissions:
-  contents: read
+  contents: write
 
 jobs:
   release:
@@ -20,21 +21,18 @@ jobs:
     steps:
       - name: Chekout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0 # fetch all commits/branches
 
       - name: Set up Python
         uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
         with:
           python-version: "3.x"
 
       - name: Configure git
-        env:
-          ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }}
         run: |
-          git config --global url."https://percona-platform-robot:${ROBOT_TOKEN}@github.com".insteadOf "https://github.com"
           git config user.name "GitHub Action"
           git config user.email "[email protected]"
-          git config user.password "${ROBOT_TOKEN}"
-          echo "GIT_USER=percona-platform-robot:${ROBOT_TOKEN}" >> $GITHUB_ENV
 
       - name: Install MkDocs
         run: |
@@ -44,6 +42,6 @@ jobs:
 
       - name: Deploy
         run: |
-          mike deploy main -p
           mike set-default main -p
           mike retitle main "Beta" -p
+          mike deploy main -p

.github/workflows/postgresql-16-ppg-package-pgxs.yml

@@ -107,7 +107,7 @@ jobs:
         working-directory: src/pg_tde
 
       - name: Report on test fail
-        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
         if: ${{ failure() }}
         with:
           name: Regressions diff and postgresql log
@@ -130,7 +130,7 @@ jobs:
           sudo cp /usr/lib/postgresql/16/lib/pg_tde* pgtde-ppg16/usr/lib/postgresql/16/lib/
 
       - name: Upload tgz
-        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
         with:
           name: pg_tde_ppg16_binary
           path: pgtde-ppg16
@@ -152,7 +152,7 @@ jobs:
           sudo dpkg -i --debug=7777 pgtde-ppg16.deb
 
       - name: Upload deb
-        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
         with:
           name: pg_tde_deb
           path: pgtde-ppg16.deb

.github/workflows/postgresql-16-src-make-ssl11.yml

@@ -97,7 +97,7 @@ jobs:
         working-directory: src/contrib/pg_tde
 
       - name: Report on test fail
-        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
         if: ${{ failure() }}
         with:
           name: Regressions diff and postgresql log

.github/workflows/postgresql-16-src-make.yml

@@ -97,7 +97,7 @@ jobs:
         working-directory: src/contrib/pg_tde
 
       - name: Report on test fail
-        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
         if: ${{ failure() }}
         with:
           name: Regressions diff and postgresql log

.github/workflows/postgresql-16-src-meson.yml

@@ -85,7 +85,7 @@ jobs:
         working-directory: src/build
 
       - name: Report on test fail
-        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
         if: ${{ failure() }}
         with:
           name: Regressions diff and postgresql log

.github/workflows/postgresql-17-src-make.yml

@@ -97,7 +97,7 @@ jobs:
         working-directory: src/contrib/pg_tde
 
       - name: Report on test fail
-        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
         if: ${{ failure() }}
         with:
           name: Regressions diff and postgresql log

.github/workflows/postgresql-17-src-meson-perf.yml

@@ -79,7 +79,7 @@ jobs:
         working-directory: src/build
 
       - name: Report on test fail
-        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
         if: ${{ failure() }}
         with:
           name: Regressions diff and postgresql log
@@ -139,7 +139,7 @@ jobs:
           echo "EOF" >> $GITHUB_ENV
         working-directory: inst
 
-      - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+      - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
         with:
           name: pr_perf_results
           path: inst/pr_perf_results

.github/workflows/postgresql-17-src-meson.yml

@@ -80,7 +80,7 @@ jobs:
         working-directory: src/build
 
       - name: Report on test fail
-        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
         if: ${{ failure() }}
         with:
           name: Regressions diff and postgresql log

.github/workflows/postgresql-perf-results.yml

@@ -8,6 +8,7 @@ on:
 
 permissions:
   contents: read
+  pull-requests: write
 
 jobs:
   download:
@@ -38,17 +39,10 @@ jobs:
         run: |
           unzip pr_perf_results.zip
 
-      - name: Clone pg_tde repository
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-        with:
-          path: 'src'
-          ref: ${{ github.event.workflow_run.head_branch }}
-
       - name: 'Create comment'
         run: |
-          gh pr comment ${PR_NUMBER} -F ../pr_perf_results --edit-last || \
-            gh pr comment ${PR_NUMBER} -F ../pr_perf_results
+          gh pr comment ${PR_NUMBER} -F pr_perf_results --repo ${{ github.repository }}  --edit-last || \
+            gh pr comment ${PR_NUMBER} -F pr_perf_results --repo ${{ github.repository }}
         env:
           PR_NUMBER: ${{ github.event.number }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        working-directory: src

.github/workflows/postgresql-pgdg-package-pgxs.yml

@@ -104,7 +104,7 @@ jobs:
         working-directory: src/pg_tde
 
       - name: Report on test fail
-        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
         if: ${{ failure() }}
         with:
           name: Regressions diff and postgresql log
@@ -131,7 +131,7 @@ jobs:
       - name: Upload tgz
         env:
           POSTGRESQL_VERSION: ${{ matrix.postgresql-version }}
-        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
         with:
           name: pg_tde_pgdg$POSTGRESQL_VERSION_binary
           path: pgtde-pgdg$POSTGRESQL_VERSION
@@ -159,7 +159,7 @@ jobs:
       - name: Upload deb
         env:
           POSTGRESQL_VERSION: ${{ matrix.postgresql-version }}
-        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
         with:
           name: pg_tde_deb
           path: pgtde-pgdg$POSTGRESQL_VERSION.deb

.github/workflows/scorecard.yml

@@ -35,14 +35,14 @@ jobs:
           publish_results: true
 
       - name: Upload results
-        uses: actions/upload-artifact@97a0fba1372883ab732affbe8f94b823f91727db # v3.pre.node20
+        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
         with:
           name: SARIF file
           path: results.sarif
           retention-days: 5
 
       # Upload the results to GitHub's code scanning dashboard (optional).
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@aa578102511db1f4524ed59b8cc2bae4f6e88195 # v3.27.6
+        uses: github/codeql-action/upload-sarif@48ab28a6f5dbc2a99bf1e0131198dd8f1df78169 # v3.28.0
         with:
           sarif_file: results.sarif

Makefile

@@ -3,7 +3,7 @@
 PGFILEDESC = "pg_tde access method"
 MODULE_big = pg_tde
 EXTENSION = pg_tde
-DATA = pg_tde--1.0.sql
+DATA = pg_tde--1.0-beta2.sql
 
 REGRESS_OPTS = --temp-config $(top_srcdir)/contrib/pg_tde/pg_tde.conf
 REGRESS = toast_decrypt_basic \

documentation/docs/faq.md

@@ -0,0 +1,30 @@
+# FAQ
+
+## Why do I need TDE?
+
+- Compliance to security and legal regulations like GDPR, PCI DSS and others
+- Encryption of backups
+- Granular encryption of specific data sets and reducing the performance overhead that encryption brings
+- Additional layer of security to existing security measures
+
+## I use disk-level encryption. Why should I care about TDE?
+
+Encrypting a hard drive encrypts all data including system and application files that are there. However, disk encryption doesn’t protect your data after the boot-up of your system. During runtime, the files are decrypted with disk-encryption.
+
+TDE focuses specifically on data files and offers a more granular control over encrypted data. It also ensures that files are encrypted on disk during runtime and when moved to another system or storage.
+
+Consider using TDE and storage-level encryption together to add another layer of data security
+
+## Is TDE enough to ensure data security?
+
+No. TDE is an additional layer to ensure data security. It protects data at rest. Consider introducing also these measures:
+
+* Access control and authentication
+* Strong network security like TLS
+* Disk encryption
+* Regular monitoring and auditing
+* Additional data protection for sensitive fields (e.g., application-layer encryption)
+
+## What happens to my data if I lose a principal key?
+
+If you lose encryption keys, especially, the principal key, the data is lost. That's why it's critical to back up your encryption keys securely.

documentation/docs/functions.md

@@ -21,7 +21,7 @@ Creates a new key provider for the database using a remote HashiCorp Vault serve
 The specified access parameters require permission to read and write keys at the location.
 
 ```
-SELECT pg_tde_add_key_provider_vault_v2('provider-name',:'secret_token','url','mount','ca_path');
+SELECT pg_tde_add_key_provider_vault_v2('provider-name','secret_token','url','mount','ca_path');
 ```
 
 where:
@@ -33,6 +33,24 @@ where:
 
 All parameters can be either strings, or JSON objects [referencing remote parameters](external-parameters.md).
 
+## pg_tde_add_key_provider_kmip
+
+Creates a new key provider for the database using a remote KMIP server.
+
+The specified access parameters require permission to read and write keys at the server.
+
+```
+SELECT pg_tde_add_key_provider_kmip('provider-name','kmip-IP', 5696, '/path_to/server_certificate.pem', '/path_to/client_key.pem');
+```
+
+where:
+
+* `provider-name` is the name of the provider. You can specify any name, it's for you to identify the provider.
+* `kmip-IP` is the IP address of a domain name of the KMIP server
+* The port to communicate with the KMIP server. The default port is `5696`.
+* `server-certificate` is the path to the certificate file for the KMIP server.
+* `client key` is the path to the client key.
+
 ## pg_tde_set_principal_key
 
 Sets the principal key for the database using the specified key provider.
@@ -72,12 +90,19 @@ SELECT pg_tde_rotate_principal_key('name-of-the-new-principal-key', NULL);
 SELECT pg_tde_rotate_principal_key(NULL, 'name-of-the-new-provider');
 ```
 
+
 ## pg_tde_is_encrypted
 
-Tells if a table is using the `pg_tde` access method or not.
+Tells if a table is encrypted using the `tde_heap` access method or not.
+
+To verify a table encryption, run the following statement:
 
 ```
 SELECT pg_tde_is_encrypted('table_name');
 ```
 
+You can also verify if the table in a custom schema is encrypted. Pass teh schema name for the function as follows:
 
+```
+SELECT pg_tde_is_encrypted('schema.table_name');
+```

documentation/docs/index.md

@@ -1,8 +1,10 @@
 # `pg_tde` documentation
 
-`pg_tde` is the extension that brings in [Transparent Data Encryption (TDE)](tde.md) to PostgreSQL and enables users to keep sensitive data safe and secure. The encryption is transparent for users allowing them to access and manipulate the data and not to worry about the encryption process.
+`pg_tde` is the open source PostgreSQL extension that provides Transparent Data Encryption (TDE) to protect data at rest. This ensures that the data stored on disk is encrypted, and no one can read it without the proper encryption keys, even if they gain access to the physical storage media. 
 
-Users can configure encryption differently for each database, encrypting specific tables in some databases with different encryption keys, while keeping others non encrypted. 
+You can configure encryption differently for each database, encrypting specific tables in some databases with different encryption keys while keeping others unencrypted.
+
+Lear more [what is Transparent Data Encryption](tde.md#how-does-it-work) and [why you need it](tde.md#why-do-you-need-tde).
 
 !!! important 
 
@@ -23,8 +25,14 @@ Users can configure encryption differently for each database, encrypting specifi
 
 ## Known limitations
 
-* Keys in the local keyfile are stored unencrypted. We encourage you to use Key management storage.
+* Keys in the local keyfile are stored unencrypted. For better security we recommend using the Key management storage. 
 * System tables are currently not encrypted.
+* Currently you cannot update the configuration of an existing Key Management Store (KMS). If its configuration changes (e.g. your Vault server has a new URL), you must set up a new key provider in `pg_tde` and create new keys there. Both the KMS and PostgreSQL servers must be up and running during these changes. [Reach out to our experts](https://www.percona.com/about/contact) for assistance and to outline the best update path for you.
+
+   We plan to introduce the way to update the configuration of an existing KMS in future releases. 
+
+* `pg_rewind` doesn't work with encrypted WAL for now. We plan to fix it in future releases.
+
 
 <i warning>:material-alert: Warning:</i> Note that introducing encryption/decryption affects performance. Our benchmark tests show less than 10% performance overhead for most situations. However, in some specific applications such as those using JSONB operations, performance degradation might be higher.
 
@@ -42,10 +50,9 @@ The `pg_tde` extension comes in two distinct versions with specific access metho
 
 ### Which version to chose?
 
-The answer is pretty straightforward: if you don't use indexes and don't need index encryption, use the community version and the `tde_heap_basic` access method. Check the [upstream documentation :octicons-link-external-16:](https://github.com/percona/pg_tde/blob/main/README.md) how to get started.
+The answer is pretty straightforward: for data sets where indexing is not mandatory or index encryption is not required, use the community version and the `tde_heap_basic` access method. Check the [upstream documentation :octicons-link-external-16:](https://github.com/percona/pg_tde/blob/main/README.md) how to get started.
 
 Otherwise, enjoy full encryption with the Percona Server for PostgreSQL version and the `tde_heap` access method. 
 
 Still not sure? [Contact our experts](https://www.percona.com/about/contact) to find the best solution for you.
 
-