firewalldb: add privacy mapper SQL migration

This commit introduces the migration logic for transitioning the
privacy mapper store from kvdb to SQL.

Note that as of this commit, the migration is not yet triggered by any
production code, i.e. only tests execute the migration logic.

Author: ViktorTigerstrom

firewalldb/sql_migration.go

@@ -67,6 +67,11 @@ func (e *kvEntry) namespacedKey() string {
 	return ns
 }
 
+// privacyPairs is a type alias for a map that holds the privacy pairs, where
+// the outer key is the group ID, and the value is a map of real to pseudo
+// values.
+type privacyPairs = map[int64]map[string]string
+
 // MigrateFirewallDBToSQL runs the migration of the firwalldb stores from the
 // bbolt database to a SQL database. The migration is done in a single
 // transaction to ensure that all rows in the stores are migrated or none at
@@ -84,10 +89,14 @@ func MigrateFirewallDBToSQL(ctx context.Context, kvStore *bbolt.DB,
 		return err
 	}
 
+	err = migratePrivacyMapperDBToSQL(ctx, kvStore, sqlTx)
+	if err != nil {
+		return err
+	}
+
 	log.Infof("The rules DB has been migrated from KV to SQL.")
 
-	// TODO(viktor): Add migration for the privacy mapper and the action
-	// stores.
+	// TODO(viktor): Add migration for the action stores.
 
 	return nil
 }
@@ -487,3 +496,281 @@ func verifyBktKeys(bkt *bbolt.Bucket, errorOnKeyValues bool,
 		return fmt.Errorf("unexpected key found: %s", key)
 	})
 }
+
+// migratePrivacyMapperDBToSQL runs the migration of the privacy mapper store
+// from the KV database to the SQL database. The function also asserts that the
+// migrated values match the original values in the privacy mapper store.
+func migratePrivacyMapperDBToSQL(ctx context.Context, kvStore *bbolt.DB,
+	sqlTx SQLQueries) error {
+
+	log.Infof("Starting migration of the privacy mapper store to SQL")
+
+	// 1) Collect all privacy pairs from the KV store.
+	privPairs, err := collectPrivacyPairs(ctx, kvStore, sqlTx)
+	if err != nil {
+		return fmt.Errorf("error migrating privacy mapper store: %w",
+			err)
+	}
+
+	// 2) Insert all collected privacy pairs into the SQL database.
+	err = insertPrivacyPairs(ctx, sqlTx, privPairs)
+	if err != nil {
+		return fmt.Errorf("insertion of privacy pairs failed: %w", err)
+	}
+
+	// 3) Validate that all inserted privacy pairs match the original values
+	// in the KV store. Note that this is done after all values have been
+	// inserted, to ensure that the migration doesn't overwrite any values
+	// after they were inserted.
+	err = validatePrivacyPairsMigration(ctx, sqlTx, privPairs)
+	if err != nil {
+		return fmt.Errorf("migration validation of privacy pairs "+
+			"failed: %w", err)
+	}
+
+	log.Infof("Migration of the privacy mapper stores to SQL completed. "+
+		"Total number of rows migrated: %d", len(privPairs))
+
+	return nil
+}
+
+// collectPrivacyPairs collects all privacy pairs from the KV store.
+func collectPrivacyPairs(ctx context.Context, kvStore *bbolt.DB,
+	sqlTx SQLQueries) (privacyPairs, error) {
+
+	groupPairs := make(privacyPairs)
+
+	return groupPairs, kvStore.View(func(kvTx *bbolt.Tx) error {
+		bkt := kvTx.Bucket(privacyBucketKey)
+		if bkt == nil {
+			// If we haven't generated any privacy bucket yet,
+			// we can skip the migration, as there are no privacy
+			// pairs to migrate.
+			return nil
+		}
+
+		return bkt.ForEach(func(groupId, v []byte) error {
+			if v != nil {
+				return fmt.Errorf("expected only buckets "+
+					"under %s bkt, but found value %s",
+					privacyBucketKey, v)
+			}
+
+			gBkt := bkt.Bucket(groupId)
+			if gBkt == nil {
+				return fmt.Errorf("group bkt for group id "+
+					"%s not found", groupId)
+			}
+
+			groupSqlId, err := sqlTx.GetSessionIDByAlias(
+				ctx, groupId,
+			)
+			if errors.Is(err, sql.ErrNoRows) {
+				return fmt.Errorf("session with group id %x "+
+					"not found in sql db", groupId)
+			} else if err != nil {
+				return err
+			}
+
+			groupRealToPseudoPairs, err := collectGroupPairs(gBkt)
+			if err != nil {
+				return fmt.Errorf("processing group bkt "+
+					"for group id %s (sqlID %d) failed: %w",
+					groupId, groupSqlId, err)
+			}
+
+			groupPairs[groupSqlId] = groupRealToPseudoPairs
+
+			return nil
+		})
+	})
+}
+
+// collectGroupPairs collects all privacy pairs for a specific session group,
+// i.e. the group buckets under the privacy mapper bucket in the KV store.
+// The function returns them as a map, where the key is the real value, and
+// the value for the key is the pseudo values.
+// It also checks that the pairs are consistent, i.e. that for each real value
+// there is a corresponding pseudo value, and vice versa. If the pairs are
+// inconsistent, it returns an error indicating the mismatch.
+func collectGroupPairs(bkt *bbolt.Bucket) (map[string]string, error) {
+	var (
+		realToPseudoRes map[string]string
+		pseudoToRealRes map[string]string
+		err             error
+	)
+
+	if realBkt := bkt.Bucket(realToPseudoKey); realBkt != nil {
+		realToPseudoRes, err = collectPairs(realBkt)
+		if err != nil {
+			return nil, fmt.Errorf("fetching real to pseudo pairs "+
+				"failed: %w", err)
+		}
+	} else {
+		return nil, fmt.Errorf("%s bucket not found", realToPseudoKey)
+	}
+
+	if pseudoBkt := bkt.Bucket(pseudoToRealKey); pseudoBkt != nil {
+		pseudoToRealRes, err = collectPairs(pseudoBkt)
+		if err != nil {
+			return nil, fmt.Errorf("fetching pseudo to real pairs "+
+				"failed: %w", err)
+		}
+	} else {
+		return nil, fmt.Errorf("%s bucket not found", pseudoToRealKey)
+	}
+
+	if len(realToPseudoRes) != len(pseudoToRealRes) {
+		return nil, fmt.Errorf("missmatch between nubmer of pairs in "+
+			"%s bucket (pairs found: %d) and %s bucket (pairs "+
+			"found: %d)", realToPseudoKey, len(realToPseudoRes),
+			pseudoToRealKey, len(pseudoToRealRes))
+	}
+
+	for realVal, pseudoVal := range realToPseudoRes {
+		if rv, ok := pseudoToRealRes[pseudoVal]; !ok || rv != realVal {
+			return nil, fmt.Errorf("the real value %s found in "+
+				"the %s bucket doesn't match the value %s "+
+				"found in the %s bucket",
+				realVal, realToPseudoKey, rv, pseudoToRealKey)
+		}
+	}
+
+	return realToPseudoRes, nil
+}
+
+// collectPairs collects all privacy pairs from a specific realToPseudoKey or
+// pseudoToRealKey bucket in the KV store. It returns a map where the key is
+// the real value or pseudo value, and the value is the corresponding pseudo
+// value or real value, respectively (depending on if the realToPseudo or
+// pseudoToReal bucket is passed to the function).
+func collectPairs(pairsBucket *bbolt.Bucket) (map[string]string, error) {
+	pairsRes := make(map[string]string)
+
+	return pairsRes, pairsBucket.ForEach(func(k, v []byte) error {
+		if v == nil {
+			return fmt.Errorf("expected only key-values under "+
+				"pairs bucket, but found bucket %s", k)
+		}
+
+		if len(v) == 0 {
+			return fmt.Errorf("empty value stored for privacy "+
+				"pairs key %s", k)
+		}
+
+		pairsRes[string(k)] = string(v)
+
+		return nil
+	})
+}
+
+// insertPrivacyPairs inserts the collected privacy pairs into the SQL database.
+func insertPrivacyPairs(ctx context.Context, sqlTx SQLQueries,
+	pairs privacyPairs) error {
+
+	for groupId, groupPairs := range pairs {
+		err := insertGroupPairs(ctx, sqlTx, groupId, groupPairs)
+		if err != nil {
+			return fmt.Errorf("inserting group pairs for group "+
+				"id %d failed: %w", groupId, err)
+		}
+	}
+
+	return nil
+}
+
+// insertGroupPairs inserts the privacy pairs for a specific group into
+// the SQL database. It checks for duplicates before inserting, and returns
+// an error if a duplicate pair is found. The function takes a map of real
+// to pseudo values, where the key is the real value and the value is the
+// corresponding pseudo value.
+func insertGroupPairs(ctx context.Context, sqlTx SQLQueries, groupID int64,
+	pairs map[string]string) error {
+
+	for realVal, pseudoVal := range pairs {
+		err := sqlTx.InsertPrivacyPair(
+			ctx, sqlc.InsertPrivacyPairParams{
+				GroupID:   groupID,
+				RealVal:   realVal,
+				PseudoVal: pseudoVal,
+			},
+		)
+		if err != nil {
+			return fmt.Errorf("inserting privacy pair %s:%s "+
+				"failed: %w", realVal, pseudoVal, err)
+		}
+	}
+
+	return nil
+}
+
+// validatePrivacyPairsMigration validates that the migrated privacy pairs
+// match the original values in the KV store.
+func validatePrivacyPairsMigration(ctx context.Context, sqlTx SQLQueries,
+	pairs privacyPairs) error {
+
+	for groupId, groupPairs := range pairs {
+		err := validateGroupPairsMigration(
+			ctx, sqlTx, groupId, groupPairs,
+		)
+		if err != nil {
+			return fmt.Errorf("migration validation of privacy "+
+				"pairs for group %d failed: %w", groupId, err)
+		}
+	}
+
+	return nil
+}
+
+// validateGroupPairsMigration validates that the migrated privacy pairs for
+// a specific group match the original values in the KV store. It checks that
+// for each real value, the pseudo value in the SQL database matches the
+// original pseudo value, and vice versa. If any mismatch is found, it returns
+// an error indicating the mismatch.
+func validateGroupPairsMigration(ctx context.Context, sqlTx SQLQueries,
+	groupID int64, pairs map[string]string) error {
+
+	for realVal, pseudoVal := range pairs {
+		resPseudoVal, err := sqlTx.GetPseudoForReal(
+			ctx, sqlc.GetPseudoForRealParams{
+				GroupID: groupID,
+				RealVal: realVal,
+			},
+		)
+		if errors.Is(err, sql.ErrNoRows) {
+			return fmt.Errorf("migrated privacy pair %s:%s not "+
+				"found for real value", realVal, pseudoVal)
+		}
+		if err != nil {
+			return err
+		}
+
+		if resPseudoVal != pseudoVal {
+			return fmt.Errorf("pseudo value in db %s, does not "+
+				"match original value %s, for real value %s",
+				resPseudoVal, pseudoVal, realVal)
+		}
+
+		resRealVal, err := sqlTx.GetRealForPseudo(
+			ctx, sqlc.GetRealForPseudoParams{
+				GroupID:   groupID,
+				PseudoVal: pseudoVal,
+			},
+		)
+		if errors.Is(err, sql.ErrNoRows) {
+			return fmt.Errorf("migrated privacy pair %s:%s not "+
+				"found for pseudo value", realVal, pseudoVal)
+		}
+		if err != nil {
+			return err
+		}
+
+		if resRealVal != realVal {
+			return fmt.Errorf("real value in db %s, does not "+
+				"match original value %s, for pseudo value %s",
+				resRealVal, realVal, pseudoVal)
+		}
+	}
+
+	return nil
+}