Skip to content

Commit

Permalink
PMM-12899 migrate /v1/backup/Backups
Browse files Browse the repository at this point in the history
  • Loading branch information
ademidoff committed Apr 8, 2024
1 parent c5628b9 commit 9393931
Show file tree
Hide file tree
Showing 9 changed files with 104 additions and 84 deletions.
39 changes: 21 additions & 18 deletions api/MIGRATION_EXAMPLES.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,42 +21,45 @@ curl -s -X GET http://admin:[email protected]:8080/v1/inventory/agents/02ecd9e3-d7

### POST /v1/inventory/Agents/List -> GET /v1/inventory/agents?agent_type=AGENT_TYPE_POSTGRES_EXPORTER
curl -s -X GET http://admin:[email protected]:8080/v1/inventory/agents
curl -s -X GET http://admin:admin@localhost:8080/v1/inventory/agents?agent_type=AGENT_TYPE_POSTGRES_EXPORTER
curl -s -X GET http://admin:admin@localhost:8080/v1/inventory/agents?agent_type=AGENT_TYPE_PMM_AGENT
curl -s -X GET http://admin:admin@localhost:8080/v1/inventory/agents?pmm_agent_id=pmm-server
curl -s -X GET http://admin:admin@localhost:8080/v1/inventory/agents?pmm_agent_id=/agent_id/02ecd9e3-d7b8-4d94-9c75-060b8e6e3e84
curl -s -X GET http://admin:admin@localhost:8080/v1/inventory/agents?pmm_agent_id=02ecd9e3-d7b8-4d94-9c75-060b8e6e3e84
curl -s -X GET http://admin:admin@localhost:8080/v1/inventory/agents?service_id=/service_id/6984244c-0a18-4508-a219-3977e8fb01d0
curl -s -X GET http://admin:admin@localhost:8080/v1/inventory/agents?service_id=6984244c-0a18-4508-a219-3977e8fb01d0
curl -s -X GET http://admin:admin@127.0.0.1:8080/v1/inventory/agents?agent_type=AGENT_TYPE_POSTGRES_EXPORTER
curl -s -X GET http://admin:admin@127.0.0.1:8080/v1/inventory/agents?agent_type=AGENT_TYPE_PMM_AGENT
curl -s -X GET http://admin:admin@127.0.0.1:8080/v1/inventory/agents?pmm_agent_id=pmm-server
curl -s -X GET http://admin:admin@127.0.0.1:8080/v1/inventory/agents?pmm_agent_id=/agent_id/02ecd9e3-d7b8-4d94-9c75-060b8e6e3e84
curl -s -X GET http://admin:admin@127.0.0.1:8080/v1/inventory/agents?pmm_agent_id=02ecd9e3-d7b8-4d94-9c75-060b8e6e3e84
curl -s -X GET http://admin:admin@127.0.0.1:8080/v1/inventory/agents?service_id=/service_id/6984244c-0a18-4508-a219-3977e8fb01d0
curl -s -X GET http://admin:admin@127.0.0.1:8080/v1/inventory/agents?service_id=6984244c-0a18-4508-a219-3977e8fb01d0

### POST /v1/inventory/Agents/GetLogs - GET /v1/inventory/agents/{agent_id}/logs
curl -s -X GET http://admin:admin@localhost:8080/v1/inventory/agents/49bef198-299c-41b3-ba05-578defe63678/logs
curl -s -X GET http://admin:admin@localhost:8080/v1/inventory/agents/49bef198-299c-41b3-ba05-578defe63678/logs?limit=10
curl -s -X GET http://admin:admin@127.0.0.1:8080/v1/inventory/agents/49bef198-299c-41b3-ba05-578defe63678/logs
curl -s -X GET http://admin:admin@127.0.0.1:8080/v1/inventory/agents/49bef198-299c-41b3-ba05-578defe63678/logs?limit=10

### POST /v1/inventory/Nodes/Get -> GET /v1/inventory/nodes/{node_id}
curl -s -X GET http://admin:admin@localhost:8080/v1/inventory/nodes/32c914d1-daf0-468a-aa9d-4ebb65ab2ee9
curl -s -X GET http://admin:admin@127.0.0.1:8080/v1/inventory/nodes/32c914d1-daf0-468a-aa9d-4ebb65ab2ee9

### POST /v1/inventory/Services/Get -> GET /v1/inventory/services/{service_id}
curl -s -X GET http://admin:admin@localhost:8080/v1/inventory/services/d4dfdccf-c07c-48a6-a101-b119b04d880f
curl -s -X GET http://admin:admin@127.0.0.1:8080/v1/inventory/services/d4dfdccf-c07c-48a6-a101-b119b04d880f

### POST /v1/inventory/Services/List -> GET /v1/inventory/services
curl -s -X GET http://admin:admin@localhost:8080/v1/inventory/services
curl -s -X GET http://admin:admin@127.0.0.1:8080/v1/inventory/services

### POST /v1/inventory/Services/Change -> PUT /v1/inventory/services/{service_id}
curl -s -X PUT -d '{"cluster": "test2","environment":"dev","replication_set":"main"}' http://admin:admin@localhost:8080/v1/inventory/services/d4dfdccf-c07c-48a6-a101-b119b04d880f
curl -s -X PUT -d '{"cluster": "test2","environment":"dev","replication_set":"main"}' http://admin:admin@127.0.0.1:8080/v1/inventory/services/d4dfdccf-c07c-48a6-a101-b119b04d880f
### add/update custom labels
curl -s -X PUT -d '{"custom_labels":{"values":{"env":"foo","bar":"123"}}}' http://admin:admin@localhost:8080/v1/inventory/services/d4dfdccf-c07c-48a6-a101-b119b04d880f
curl -s -X PUT -d '{"custom_labels":{"values":{"env":"foo","bar":"123"}}}' http://admin:admin@127.0.0.1:8080/v1/inventory/services/d4dfdccf-c07c-48a6-a101-b119b04d880f
### remove a standard label and all custom labels
curl -s -X PUT -d '{"replication_set":"","custom_labels":{}}' http://admin:admin@localhost:8080/v1/inventory/services/d4dfdccf-c07c-48a6-a101-b119b04d880f
curl -s -X PUT -d '{"replication_set":"","custom_labels":{}}' http://admin:admin@127.0.0.1:8080/v1/inventory/services/d4dfdccf-c07c-48a6-a101-b119b04d880f

### POST /v1/inventory/Services/ListTypes -> POST /v1/inventory/services:getTypes
curl -s -X POST http://admin:admin@localhost:8080/v1/inventory/services:getTypes
curl -s -X POST http://admin:admin@127.0.0.1:8080/v1/inventory/services:getTypes

### /v1/management/Service/Remove -> DELETE /v1/management/services/{service_id}
curl -s -X DELETE http://admin:admin@localhost:8080/v1/management/services/b7d3b87a-d366-4cb4-b101-03d68f73a7c0
curl -s -X DELETE http://admin:admin@127.0.0.1:8080/v1/management/services/b7d3b87a-d366-4cb4-b101-03d68f73a7c0
### pmm-admin remove mongodb mongo-svc
### pmm-admin remove mongodb mongo-svc --service-id=/service_id/ed322782-e6fd-4ad9-8ee6-a7d47b62de41
### pmm-admin remove mongodb --service-id=/service_id/ed322782-e6fd-4ad9-8ee6-a7d47b62de41

# POST /v1/advisors/FailedChecks -> GET /v1/advisors/checks/failed
curl -s -X GET "http://admin:admin@localhost:8080/v1/advisors/checks/failed?service_id=bcc83096-b006-4d2e-ac17-365a57c3e37a&page_size=100"
curl -s -X GET "http://admin:[email protected]:8080/v1/advisors/checks/failed?service_id=bcc83096-b006-4d2e-ac17-365a57c3e37a&page_size=100"

# POST /v1/backup/Artifacts/PITRTimeranges -> GET /v1/backups/artifacts/{artifact_id}/pitr-timeranges
curl -s -X GET http://admin:[email protected]:8080/v1/backups/artifacts/bcc83096-b006-4d2e-ac17-365a57c3e37a/pitr-timeranges
16 changes: 8 additions & 8 deletions api/MIGRATION_TO_V3.md
Original file line number Diff line number Diff line change
Expand Up @@ -96,14 +96,14 @@ POST /v1/backup/Artifacts/Delete DELETE /v1/backups/artifacts
POST /v1/backup/Artifacts/PITRTimeranges GET /v1/backups/artifacts/{artifact_id}/pitr-timeranges ✅

**BackupsService** **BackupService** NOTE: BackupsService renamed to BackupService
POST /v1/backup/Backups/ChangeScheduled PUT /v1/backups:changeScheduled
POST /v1/backup/Backups/GetLogs GET /v1/backups/{id}/logs
POST /v1/backup/Backups/ListArtifactCompatibleServices GET /v1/backups/{id}/compatible-services
POST /v1/backup/Backups/ListScheduled GET /v1/backups/scheduled
POST /v1/backup/Backups/RemoveScheduled GET /v1/backups/scheduled/{id}
POST /v1/backup/Backups/ChangeScheduled PUT /v1/backups:changeScheduled
POST /v1/backup/Backups/GetLogs GET /v1/backups/{artifact_id}/logs
POST /v1/backup/Backups/ListArtifactCompatibleServices GET /v1/backups/{artifact_id}/compatible-services
POST /v1/backup/Backups/ListScheduled GET /v1/backups/scheduled
POST /v1/backup/Backups/RemoveScheduled DELETE /v1/backups/scheduled/{scheduled_backup_id} ✅
POST /v1/backup/Backups/Restore NOTE: Moved to RestoreService
POST /v1/backup/Backups/Schedule POST /v1/backups:schedule
POST /v1/backup/Backups/Start POST /v1/backups:start
POST /v1/backup/Backups/Schedule POST /v1/backups:schedule
POST /v1/backup/Backups/Start POST /v1/backups:start

**LocationsService** **LocationsService**
POST /v1/backup/Locations/Add POST /v1/backups/locations
Expand Down Expand Up @@ -184,4 +184,4 @@ The custom method should be used in the following cases:
1. When the action cannot be performed by the standard RESTful methods.
2. When the action performed is not idempotent.
3. When the action performed manipulates data, but does not fit into the standard CRUD operations.
4. When the action performed might contain sensitive data, that cannot be passed via URL query params.
4. When the action performed contains sensitive data, that cannot be passed via URL query params.
17 changes: 7 additions & 10 deletions api/backup/v1/backup.proto
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@ service BackupService {
// StartBackup request backup specified service to location.
rpc StartBackup(StartBackupRequest) returns (StartBackupResponse) {
option (google.api.http) = {
post: "/v1/backup/Backups/Start"
post: "/v1/backups:start"
body: "*"
};
option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {
Expand All @@ -186,8 +186,7 @@ service BackupService {
// ListArtifactCompatibleServices lists compatible services for restoring a backup.
rpc ListArtifactCompatibleServices(ListArtifactCompatibleServicesRequest) returns (ListArtifactCompatibleServicesResponse) {
option (google.api.http) = {
post: "/v1/backup/Backups/ListArtifactCompatibleServices"
body: "*"
get: "/v1/backups/{artifact_id}/compatible-services"
};
option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {
summary: "List Compatible Services"
Expand All @@ -197,7 +196,7 @@ service BackupService {
// ScheduleBackup schedules repeated backup.
rpc ScheduleBackup(ScheduleBackupRequest) returns (ScheduleBackupResponse) {
option (google.api.http) = {
post: "/v1/backup/Backups/Schedule"
post: "/v1/backups:schedule"
body: "*"
};
option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {
Expand All @@ -208,8 +207,7 @@ service BackupService {
// ListScheduledBackups returns all scheduled backups.
rpc ListScheduledBackups(ListScheduledBackupsRequest) returns (ListScheduledBackupsResponse) {
option (google.api.http) = {
post: "/v1/backup/Backups/ListScheduled"
body: "*"
get: "/v1/backups/scheduled"
};
option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {
summary: "List Scheduled Backups"
Expand All @@ -219,7 +217,7 @@ service BackupService {
// ChangeScheduledBackup changes existing scheduled backup.
rpc ChangeScheduledBackup(ChangeScheduledBackupRequest) returns (ChangeScheduledBackupResponse) {
option (google.api.http) = {
post: "/v1/backup/Backups/ChangeScheduled"
put: "/v1/backups:changeScheduled"
body: "*"
};
option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {
Expand All @@ -230,8 +228,7 @@ service BackupService {
// RemoveScheduledBackup removes existing scheduled backup.
rpc RemoveScheduledBackup(RemoveScheduledBackupRequest) returns (RemoveScheduledBackupResponse) {
option (google.api.http) = {
post: "/v1/backup/Backups/RemoveScheduled"
body: "*"
delete: "/v1/backups/{scheduled_backup_id}"
};
option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {
summary: "Remove a Scheduled Backup"
Expand All @@ -241,7 +238,7 @@ service BackupService {
// GetLogs returns logs from the underlying tools for a backup/restore job.
rpc GetLogs(GetLogsRequest) returns (GetLogsResponse) {
option (google.api.http) = {
post: "/v1/backup/Backups/GetLogs"
post: "/v1/backups/{artifact_id}/logs"
body: "*"
};
option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {
Expand Down
53 changes: 34 additions & 19 deletions managed/models/agent_helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,13 @@ import (
)

const (
agentIDPrefix = "/agent_id/"
serviceIDPrefix = "/service_id/"
nodeIDPrefix = "/node_id/"
actionIDPrefix = "/action_id/"
agentIDPrefix = "/agent_id/"
serviceIDPrefix = "/service_id/"
nodeIDPrefix = "/node_id/"
actionIDPrefix = "/action_id/"
scheduledTaskIDPrefix = "/scheduled_task_id/"
artifactIDPrefix = "/artifact_id/"
restoreIDPrefix = "/restore_id/"
)

// MySQLOptionsParams contains methods to create MySQLOptions object.
Expand Down Expand Up @@ -170,38 +173,50 @@ func AzureOptionsFromRequest(params AzureOptionsParams) *AzureOptions {

// NormalizeAgentID adds a prefix to the agent ID if it does not already contain it.
func NormalizeAgentID(agentID string) string {
if agentID == "" || agentID == "pmm-server" || strings.HasPrefix(agentID, agentIDPrefix) {
if agentID == "pmm-server" {
return agentID
}

return agentIDPrefix + agentID
return normalizeID(agentID, agentIDPrefix)
}

// NormalizeServiceID adds a prefix to the service ID if it does not already contain it.
func NormalizeServiceID(serviceID string) string {
if serviceID == "" || strings.HasPrefix(serviceID, serviceIDPrefix) {
return serviceID
}

return serviceIDPrefix + serviceID
return normalizeID(serviceID, serviceIDPrefix)
}

// NormalizeNodeID adds a prefix to the node ID if it does not already contain it.
func NormalizeNodeID(nodeID string) string {
if nodeID == "" || strings.HasPrefix(nodeID, nodeIDPrefix) {
return nodeID
}

return nodeIDPrefix + nodeID
return normalizeID(nodeID, nodeIDPrefix)
}

// NormalizeActionID adds a prefix to the node ID if it does not already contain it.
func NormalizeActionID(actionID string) string {
if actionID == "" || strings.HasPrefix(actionID, nodeIDPrefix) {
return actionID
return normalizeID(actionID, nodeIDPrefix)
}

// NormalizeScheduledTaskID adds a prefix to the sheduled task ID if it does not already contain it.
func NormalizeScheduledTaskID(sTaskID string) string {
return normalizeID(sTaskID, scheduledTaskIDPrefix)
}

// NormalizeArtifactID adds a prefix to the artifact ID if it does not already contain it.
func NormalizeArtifactID(artifactID string) string {
return normalizeID(artifactID, artifactIDPrefix)
}

// NormalizeRestoreID adds a prefix to the restore ID if it does not already contain it.
func NormalizeRestoreID(artifactID string) string {
return normalizeID(artifactID, restoreIDPrefix)
}

// normalizeID adds a prefix to ID if it does not already contain it.
func normalizeID(id, prefix string) string {
if id == "" || strings.HasPrefix(id, prefix) {
return id
}

return nodeIDPrefix + actionID
return prefix + id
}

func checkUniqueAgentID(q *reform.Querier, id string) error {
Expand Down
6 changes: 4 additions & 2 deletions managed/services/backup/removal_service.go
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,7 @@ func (s *RemovalService) lockArtifact(artifactID string, lockingStatus models.Ba
err error
)

if errTx := s.db.InTransactionContext(s.db.Querier.Context(), &sql.TxOptions{Isolation: sql.LevelSerializable}, func(tx *reform.TX) error {
errTx := s.db.InTransactionContext(s.db.Querier.Context(), &sql.TxOptions{Isolation: sql.LevelSerializable}, func(tx *reform.TX) error {
artifact, err = models.FindArtifactByID(tx.Querier, artifactID)
if err != nil {
return err
Expand Down Expand Up @@ -218,7 +218,9 @@ func (s *RemovalService) lockArtifact(artifactID string, lockingStatus models.Ba
}

return nil
}); errTx != nil {
})

if errTx != nil {
return nil, "", errTx
}

Expand Down
20 changes: 8 additions & 12 deletions managed/services/management/backup/artifacts_service.go
Original file line number Diff line number Diff line change
Expand Up @@ -106,11 +106,9 @@ func (s *ArtifactsService) ListArtifacts(context.Context, *backuppb.ListArtifact
}

// DeleteArtifact deletes specified artifact and its files.
func (s *ArtifactsService) DeleteArtifact(
ctx context.Context, //nolint:revive
req *backuppb.DeleteArtifactRequest,
) (*backuppb.DeleteArtifactResponse, error) {
artifact, err := models.FindArtifactByID(s.db.Querier, req.ArtifactId)
func (s *ArtifactsService) DeleteArtifact(ctx context.Context, req *backuppb.DeleteArtifactRequest) (*backuppb.DeleteArtifactResponse, error) { //nolint:revive
artifactID := models.NormalizeArtifactID(req.ArtifactId)
artifact, err := models.FindArtifactByID(s.db.Querier, artifactID)
if err != nil {
return nil, err
}
Expand All @@ -122,24 +120,22 @@ func (s *ArtifactsService) DeleteArtifact(

storage := backup.GetStorageForLocation(location)

if err := s.removalSVC.DeleteArtifact(storage, req.ArtifactId, req.RemoveFiles); err != nil {
if err := s.removalSVC.DeleteArtifact(storage, artifactID, req.RemoveFiles); err != nil {
return nil, err
}
return &backuppb.DeleteArtifactResponse{}, nil
}

// ListPitrTimeranges lists available PITR timelines/time-ranges (for MongoDB).
func (s *ArtifactsService) ListPitrTimeranges(
ctx context.Context,
req *backuppb.ListPitrTimerangesRequest,
) (*backuppb.ListPitrTimerangesResponse, error) {
func (s *ArtifactsService) ListPitrTimeranges(ctx context.Context, req *backuppb.ListPitrTimerangesRequest) (*backuppb.ListPitrTimerangesResponse, error) {
var artifact *models.Artifact
var err error

artifact, err = models.FindArtifactByID(s.db.Querier, req.ArtifactId)
artifactID := models.NormalizeArtifactID(req.ArtifactId)
artifact, err = models.FindArtifactByID(s.db.Querier, artifactID)
if err != nil {
if errors.Is(err, models.ErrNotFound) {
return nil, status.Errorf(codes.NotFound, "Artifact with ID %q not found.", req.ArtifactId)
return nil, status.Errorf(codes.NotFound, "Artifact with ID '%s' not found.", artifactID)
}
return nil, err
}
Expand Down
5 changes: 3 additions & 2 deletions managed/services/management/backup/artifacts_service_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -101,11 +101,12 @@ func TestListPitrTimeranges(t *testing.T) {
})

t.Run("fails for invalid artifact ID", func(t *testing.T) {
unknownID := "artifact_id/" + uuid.New().String()
unknownID := uuid.New().String()
response, err := artifactsService.ListPitrTimeranges(ctx, &backuppb.ListPitrTimerangesRequest{
ArtifactId: unknownID,
})
tests.AssertGRPCError(t, status.New(codes.NotFound, fmt.Sprintf("Artifact with ID %q not found.", unknownID)), err)
unknownID = models.NormalizeArtifactID(unknownID)
tests.AssertGRPCError(t, status.New(codes.NotFound, fmt.Sprintf("Artifact with ID '%s' not found.", unknownID)), err)
assert.Nil(t, response)
})

Expand Down
16 changes: 9 additions & 7 deletions managed/services/management/backup/backups_service.go
Original file line number Diff line number Diff line change
Expand Up @@ -393,7 +393,8 @@ func (s *BackupService) ChangeScheduledBackup(ctx context.Context, req *backuppb

// RemoveScheduledBackup stops and removes existing scheduled backup task.
func (s *BackupService) RemoveScheduledBackup(ctx context.Context, req *backuppb.RemoveScheduledBackupRequest) (*backuppb.RemoveScheduledBackupResponse, error) {
task, err := models.FindScheduledTaskByID(s.db.Querier, req.ScheduledBackupId)
scheduledBackupId := models.NormalizeScheduledTaskID(req.ScheduledBackupId)
task, err := models.FindScheduledTaskByID(s.db.Querier, scheduledBackupId)
if err != nil {
return nil, err
}
Expand All @@ -409,9 +410,9 @@ func (s *BackupService) RemoveScheduledBackup(ctx context.Context, req *backuppb
return nil, errors.Errorf("non-backup task: %s", task.Type)
}

errTx := s.db.InTransaction(func(tx *reform.TX) error {
errTx := s.db.InTransactionContext(ctx, nil, func(tx *reform.TX) error {
artifacts, err := models.FindArtifacts(tx.Querier, models.ArtifactFilters{
ScheduleID: req.ScheduledBackupId,
ScheduleID: scheduledBackupId,
})
if err != nil {
return err
Expand All @@ -426,7 +427,7 @@ func (s *BackupService) RemoveScheduledBackup(ctx context.Context, req *backuppb
}
}

return s.scheduleService.Remove(req.ScheduledBackupId)
return s.scheduleService.Remove(scheduledBackupId)
})
if errTx != nil {
return nil, errTx
Expand Down Expand Up @@ -456,9 +457,9 @@ func (s *BackupService) GetLogs(_ context.Context, req *backuppb.GetLogsRequest)

switch {
case req.ArtifactId != "":
jobsFilter.ArtifactID = req.ArtifactId
jobsFilter.ArtifactID = models.NormalizeArtifactID(req.ArtifactId)
case req.RestoreId != "":
jobsFilter.RestoreID = req.RestoreId
jobsFilter.RestoreID = models.NormalizeRestoreID(req.RestoreId)
default:
return nil, status.Error(codes.InvalidArgument, "One of artifact ID or restore ID is required")
}
Expand Down Expand Up @@ -509,7 +510,8 @@ func (s *BackupService) ListArtifactCompatibleServices(
ctx context.Context,
req *backuppb.ListArtifactCompatibleServicesRequest,
) (*backuppb.ListArtifactCompatibleServicesResponse, error) {
compatibleServices, err := s.compatibilityService.FindArtifactCompatibleServices(ctx, req.ArtifactId)
artifactID := models.NormalizeActionID(req.ArtifactId)
compatibleServices, err := s.compatibilityService.FindArtifactCompatibleServices(ctx, artifactID)
switch {
case err == nil:
case errors.Is(err, models.ErrNotFound):
Expand Down
Loading

0 comments on commit 9393931

Please sign in to comment.