From a6139aefb1936f45f56e3f2930acdfbc3d1d1fb3 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 19 Jun 2026 06:51:30 -0400 Subject: [PATCH 1/2] feat: add durable backup history --- CHANGELOG.md | 5 + README.md | 13 ++ internal/backup/backup.go | 416 +++++++++++---------------------- internal/backup/backup_test.go | 182 +++++++++++++-- internal/backup/config.go | 3 + internal/backup/crypto.go | 12 - internal/backup/git.go | 98 +++----- internal/backup/history.go | 69 ++++++ internal/cli/cli.go | 63 +++-- internal/store/store.go | 7 +- 10 files changed, 477 insertions(+), 391 deletions(-) create mode 100644 internal/backup/history.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e2dcb3..d34ced4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ - Archive Telegram contact records from local Postbox imports. (#7; thanks @joshp123) - Expose Telegram contacts through the crawlkit `contact-export` metadata command for Clawdex imports. (#9; thanks @joshp123) +- Add named Git backup snapshots, history listing, and non-mutating historical restores through `backup pull --ref`. + +### Changed + +- Move encrypted snapshot, Git history/tag/ref, contact export, and safe FTS query mechanics to CrawlKit while preserving the archive schema, backup manifest format, and CLI JSON contracts. ### Fixed diff --git a/README.md b/README.md index 4ae3861..01c6437 100644 --- a/README.md +++ b/README.md @@ -195,6 +195,7 @@ Inspect backup metadata: ```bash telecrawl backup status +telecrawl backup snapshots ``` Restore into the current archive DB: @@ -204,6 +205,18 @@ telecrawl backup pull telecrawl status ``` +Every changed backup is a Git commit. Add a non-moving, visible checkpoint tag +when needed, then restore that tag, commit, or branch without switching the +backup checkout: + +```bash +telecrawl backup push --tag snapshot/before-migration +telecrawl --db /tmp/telecrawl-history.db backup pull --ref snapshot/before-migration +``` + +`backup snapshots --limit N` lists recent manifest-changing commits and tags. +Keep tag names non-sensitive because Git metadata is not encrypted. + Restore into a throwaway DB for validation: ```bash diff --git a/internal/backup/backup.go b/internal/backup/backup.go index cb15d6b..cffb594 100644 --- a/internal/backup/backup.go +++ b/internal/backup/backup.go @@ -1,23 +1,20 @@ package backup import ( - "bufio" - "bytes" "context" - "encoding/json" "fmt" "os" - "path" "path/filepath" - "reflect" "sort" "strings" "time" + ckbackup "github.com/openclaw/crawlkit/backup" + "github.com/openclaw/crawlkit/mirror" "github.com/openclaw/telecrawl/internal/store" ) -const formatVersion = 1 +const formatVersion = ckbackup.FormatVersion type Manifest struct { Format int `json:"format"` @@ -39,13 +36,7 @@ type Counts struct { Messages int `json:"messages"` } -type ShardEntry struct { - Table string `json:"table"` - Path string `json:"path"` - Rows int `json:"rows"` - SHA256 string `json:"sha256"` - Bytes int64 `json:"bytes"` -} +type ShardEntry = ckbackup.ShardEntry type Result struct { Repo string `json:"repo"` @@ -53,6 +44,8 @@ type Result struct { Encrypted bool `json:"encrypted"` Shards int `json:"shards"` Messages int `json:"messages"` + Ref string `json:"ref,omitempty"` + Tag string `json:"tag,omitempty"` } func Init(ctx context.Context, opts Options) (Config, string, error) { @@ -95,6 +88,9 @@ func Push(ctx context.Context, st *store.Store, opts Options) (Result, error) { if err := ensureRepo(ctx, cfg); err != nil { return Result{}, err } + if err := validateSnapshotTag(ctx, cfg.Repo, opts.Tag); err != nil { + return Result{}, err + } if err := writeBackupReadme(cfg.Repo); err != nil { return Result{}, err } @@ -107,11 +103,21 @@ func Push(ctx context.Context, st *store.Store, opts Options) (Result, error) { if err != nil { return Result{}, err } - changed, err := commitAndPush(ctx, cfg, "sync: update encrypted telecrawl backup", opts.Push) + pushWithTag := opts.Push && strings.TrimSpace(opts.Tag) != "" + changed, err := commitAndPush(ctx, cfg, "sync: update encrypted telecrawl backup", opts.Push && !pushWithTag) if err != nil { return Result{}, err } - return Result{Repo: cfg.Repo, Changed: changed, Encrypted: true, Shards: len(manifest.Shards), Messages: manifest.Counts.Messages}, nil + tag, err := tagSnapshot(ctx, cfg, opts.Tag) + if err != nil { + return Result{}, err + } + if pushWithTag { + if err := mirror.PushAtomic(ctx, mirrorOptions(cfg), "HEAD", "refs/tags/"+tag); err != nil { + return Result{}, err + } + } + return Result{Repo: cfg.Repo, Changed: changed, Encrypted: true, Shards: len(manifest.Shards), Messages: manifest.Counts.Messages, Tag: tag}, nil } func Pull(ctx context.Context, st *store.Store, opts Options) (Result, error) { @@ -119,24 +125,37 @@ func Pull(ctx context.Context, st *store.Store, opts Options) (Result, error) { if err != nil { return Result{}, err } - if err := ensureRepo(ctx, cfg); err != nil { + ensure := ensureRepo + if strings.TrimSpace(opts.Ref) != "" { + ensure = ensureRepoForRead + } + if err := ensure(ctx, cfg); err != nil { return Result{}, err } - manifest, err := readManifest(cfg.Repo) + manifest, ref, err := readManifestAtRef(ctx, cfg.Repo, opts.Ref) if err != nil { return Result{}, err } - data, err := readSnapshot(cfg, manifest) + var data store.SnapshotData + if ref == "" { + data, err = readSnapshot(cfg, manifest) + } else { + data, err = readSnapshotAtRef(ctx, cfg, manifest, ref) + } if err != nil { return Result{}, err } if err := data.Validate(); err != nil { return Result{}, err } - if err := st.ImportSnapshot(ctx, data, "backup:"+cfg.Repo, manifest.Exported); err != nil { + sourcePath := "backup:" + cfg.Repo + if ref != "" { + sourcePath += "@" + ref + } + if err := st.ImportSnapshot(ctx, data, sourcePath, manifest.Exported); err != nil { return Result{}, err } - return Result{Repo: cfg.Repo, Changed: true, Encrypted: manifest.Encrypted, Shards: len(manifest.Shards), Messages: len(data.Messages)}, nil + return Result{Repo: cfg.Repo, Changed: true, Encrypted: manifest.Encrypted, Shards: len(manifest.Shards), Messages: len(data.Messages), Ref: ref}, nil } func Status(ctx context.Context, opts Options) (Manifest, string, error) { @@ -155,125 +174,91 @@ func Status(ctx context.Context, opts Options) (Manifest, string, error) { } func writeSnapshot(ctx context.Context, cfg Config, data store.SnapshotData, old Manifest) (Manifest, error) { - _ = ctx - recipients := normalizedStrings(cfg.Recipients) - reuseEncrypted := sameStrings(old.Recipients, recipients) - var shards []ShardEntry - add := func(table, rel string, rows any) error { - plaintext, count, err := encodeJSONL(rows) - if err != nil { - return err - } - entry, err := writeShard(cfg, old, table, rel, plaintext, count, reuseEncrypted) - if err != nil { - return err - } - shards = append(shards, entry) - return nil - } - staticTables := []struct { - table string - path string - rows any - }{ - {"contacts", "data/contacts.jsonl.gz.age", data.Contacts}, - {"chats", "data/chats.jsonl.gz.age", data.Chats}, - {"folders", "data/folders.jsonl.gz.age", data.Folders}, - {"folder_chats", "data/folder_chats.jsonl.gz.age", data.FolderChats}, - {"groups", "data/groups.jsonl.gz.age", data.Groups}, - {"group_participants", "data/group_participants.jsonl.gz.age", data.Participants}, - {"topics", "data/topics.jsonl.gz.age", data.Topics}, - } - for _, table := range staticTables { - if err := add(table.table, table.path, table.rows); err != nil { - return Manifest{}, err - } + shards := []ckbackup.Shard{ + {Table: "contacts", Path: "data/contacts.jsonl.gz.age", Rows: data.Contacts}, + {Table: "chats", Path: "data/chats.jsonl.gz.age", Rows: data.Chats}, + {Table: "folders", Path: "data/folders.jsonl.gz.age", Rows: data.Folders}, + {Table: "folder_chats", Path: "data/folder_chats.jsonl.gz.age", Rows: data.FolderChats}, + {Table: "groups", Path: "data/groups.jsonl.gz.age", Rows: data.Groups}, + {Table: "group_participants", CountKey: "participants", Path: "data/group_participants.jsonl.gz.age", Rows: data.Participants}, + {Table: "topics", Path: "data/topics.jsonl.gz.age", Rows: data.Topics}, } for _, shard := range messageShards(data.Messages) { - if err := add("messages", shard.path, shard.messages); err != nil { - return Manifest{}, err - } + shards = append(shards, ckbackup.Shard{Table: "messages", Path: shard.path, Rows: shard.messages}) } - sort.Slice(shards, func(i, j int) bool { return shards[i].Path < shards[j].Path }) - manifest := Manifest{ - Format: formatVersion, - Encrypted: true, - Exported: time.Now().UTC(), - Recipients: recipients, - Counts: Counts{ - Contacts: len(data.Contacts), - Chats: len(data.Chats), - Folders: len(data.Folders), - FolderChats: len(data.FolderChats), - Groups: len(data.Groups), - Participants: len(data.Participants), - Topics: len(data.Topics), - Messages: len(data.Messages), - }, - Shards: shards, + sharedOld := toCrawlkitManifest(old) + if len(data.Messages) == 0 { + delete(sharedOld.Counts, "messages") } - if equivalentManifest(old, manifest) { - return old, nil - } - if err := removeStaleShards(cfg.Repo, shards); err != nil { + manifest, err := ckbackup.WriteSnapshot(ctx, crawlkitConfig(cfg), shards, sharedOld) + if err != nil { return Manifest{}, err } - if err := writeManifest(cfg.Repo, manifest); err != nil { + manifest.Counts["contacts"] = len(data.Contacts) + manifest.Counts["chats"] = len(data.Chats) + manifest.Counts["folders"] = len(data.Folders) + manifest.Counts["folder_chats"] = len(data.FolderChats) + manifest.Counts["groups"] = len(data.Groups) + manifest.Counts["participants"] = len(data.Participants) + manifest.Counts["topics"] = len(data.Topics) + manifest.Counts["messages"] = len(data.Messages) + if ckbackup.EquivalentManifest(toCrawlkitManifest(old), manifest) { + return old, nil + } + if err := ckbackup.WriteManifest(cfg.Repo, manifest); err != nil { return Manifest{}, err } - return manifest, nil + return fromCrawlkitManifest(manifest), nil } func readSnapshot(cfg Config, manifest Manifest) (store.SnapshotData, error) { - if manifest.Format != formatVersion { - return store.SnapshotData{}, fmt.Errorf("unsupported backup format %d", manifest.Format) + shards, err := ckbackup.ReadSnapshot(crawlkitConfig(cfg), toCrawlkitManifest(manifest)) + if err != nil { + return store.SnapshotData{}, err } + return decodeSnapshot(shards) +} + +func decodeSnapshot(shards []ckbackup.DecodedShard) (store.SnapshotData, error) { var data store.SnapshotData - for _, shard := range manifest.Shards { - plaintext, err := decryptShardFile(cfg, shard) - if err != nil { - return store.SnapshotData{}, err - } - if got := sha256Hex(plaintext); got != shard.SHA256 { - return store.SnapshotData{}, fmt.Errorf("backup shard hash mismatch for %s", shard.Path) - } - switch shard.Table { + for _, shard := range shards { + switch shard.Entry.Table { case "contacts": - if err := decodeJSONL(plaintext, &data.Contacts); err != nil { + if err := ckbackup.DecodeJSONL(shard.Plaintext, &data.Contacts); err != nil { return store.SnapshotData{}, err } case "chats": - if err := decodeJSONL(plaintext, &data.Chats); err != nil { + if err := ckbackup.DecodeJSONL(shard.Plaintext, &data.Chats); err != nil { return store.SnapshotData{}, err } case "folders": - if err := decodeJSONL(plaintext, &data.Folders); err != nil { + if err := ckbackup.DecodeJSONL(shard.Plaintext, &data.Folders); err != nil { return store.SnapshotData{}, err } case "folder_chats": - if err := decodeJSONL(plaintext, &data.FolderChats); err != nil { + if err := ckbackup.DecodeJSONL(shard.Plaintext, &data.FolderChats); err != nil { return store.SnapshotData{}, err } case "groups": - if err := decodeJSONL(plaintext, &data.Groups); err != nil { + if err := ckbackup.DecodeJSONL(shard.Plaintext, &data.Groups); err != nil { return store.SnapshotData{}, err } case "group_participants": - if err := decodeJSONL(plaintext, &data.Participants); err != nil { + if err := ckbackup.DecodeJSONL(shard.Plaintext, &data.Participants); err != nil { return store.SnapshotData{}, err } case "topics": - if err := decodeJSONL(plaintext, &data.Topics); err != nil { + if err := ckbackup.DecodeJSONL(shard.Plaintext, &data.Topics); err != nil { return store.SnapshotData{}, err } case "messages": var messages []store.Message - if err := decodeJSONL(plaintext, &messages); err != nil { + if err := ckbackup.DecodeJSONL(shard.Plaintext, &messages); err != nil { return store.SnapshotData{}, err } data.Messages = append(data.Messages, messages...) default: - return store.SnapshotData{}, fmt.Errorf("unknown backup table %q", shard.Table) + return store.SnapshotData{}, fmt.Errorf("unknown backup table %q", shard.Entry.Table) } } sort.Slice(data.Messages, func(i, j int) bool { @@ -285,88 +270,6 @@ func readSnapshot(cfg Config, manifest Manifest) (store.SnapshotData, error) { return data, nil } -func writeShard(cfg Config, old Manifest, table, rel string, plaintext []byte, rows int, reuseEncrypted bool) (ShardEntry, error) { - hash := sha256Hex(plaintext) - path, err := resolveShardPath(cfg.Repo, rel) - if err != nil { - return ShardEntry{}, err - } - if oldEntry, ok := old.entry(rel); reuseEncrypted && ok && oldEntry.SHA256 == hash { - if info, err := os.Stat(path); err == nil { - oldEntry.Bytes = info.Size() - return oldEntry, nil - } - } - encrypted, _, err := encryptShard(plaintext, cfg.Recipients) - if err != nil { - return ShardEntry{}, err - } - if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { - return ShardEntry{}, err - } - if err := os.WriteFile(path, encrypted, 0o600); err != nil { - return ShardEntry{}, err - } - return ShardEntry{Table: table, Path: rel, Rows: rows, SHA256: hash, Bytes: int64(len(encrypted))}, nil -} - -func decryptShardFile(cfg Config, shard ShardEntry) ([]byte, error) { - path, err := resolveShardPath(cfg.Repo, shard.Path) - if err != nil { - return nil, err - } - ciphertext, err := os.ReadFile(path) // #nosec G304 -- resolveShardPath confines manifest-controlled shard paths to data/*.age inside the backup repo. - if err != nil { - return nil, err - } - return decryptShard(ciphertext, cfg.Identity) -} - -func resolveShardPath(repo, rel string) (string, error) { - clean := path.Clean(strings.TrimSpace(rel)) - if clean == "." || clean == ".." || strings.HasPrefix(clean, "../") || path.IsAbs(clean) { - return "", fmt.Errorf("backup shard path escapes backup root: %s", rel) - } - if !strings.HasPrefix(clean, "data/") || !strings.HasSuffix(clean, ".age") { - return "", fmt.Errorf("invalid backup shard path: %s", rel) - } - full := filepath.Join(repo, filepath.FromSlash(clean)) - root := filepath.Clean(filepath.Join(repo, "data")) - parent := filepath.Clean(filepath.Dir(full)) - if parent != root && !strings.HasPrefix(parent, root+string(filepath.Separator)) { - return "", fmt.Errorf("backup shard path escapes backup root: %s", rel) - } - return full, nil -} - -func encodeJSONL(rows any) ([]byte, int, error) { - value := reflect.ValueOf(rows) - if value.Kind() != reflect.Slice { - return nil, 0, fmt.Errorf("unsupported JSONL rows %T", rows) - } - var buf bytes.Buffer - enc := json.NewEncoder(&buf) - for i := 0; i < value.Len(); i++ { - if err := enc.Encode(value.Index(i).Interface()); err != nil { - return nil, 0, err - } - } - return buf.Bytes(), value.Len(), nil -} - -func decodeJSONL[T any](plaintext []byte, out *[]T) error { - scanner := bufio.NewScanner(bytes.NewReader(plaintext)) - scanner.Buffer(make([]byte, 0, 64*1024), 16*1024*1024) - for scanner.Scan() { - var value T - if err := json.Unmarshal(scanner.Bytes(), &value); err != nil { - return err - } - *out = append(*out, value) - } - return scanner.Err() -} - type messageShard struct { path string messages []store.Message @@ -404,116 +307,59 @@ func messageShards(messages []store.Message) []messageShard { } func readManifest(repo string) (Manifest, error) { - data, err := os.ReadFile(filepath.Join(repo, "manifest.json")) // #nosec G304 -- repo is the configured local backup repository. + manifest, err := ckbackup.ReadManifest(repo) if err != nil { return Manifest{}, err } - var manifest Manifest - if err := json.Unmarshal(data, &manifest); err != nil { - return Manifest{}, err - } - return manifest, nil + return fromCrawlkitManifest(manifest), nil } -func writeManifest(repo string, manifest Manifest) error { - data, err := json.MarshalIndent(manifest, "", " ") - if err != nil { - return err - } - data = append(data, '\n') - return os.WriteFile(filepath.Join(repo, "manifest.json"), data, 0o600) +func crawlkitConfig(cfg Config) ckbackup.Config { + return ckbackup.Config{Repo: cfg.Repo, Identity: cfg.Identity, Recipients: cfg.Recipients} } -func (m Manifest) entry(path string) (ShardEntry, bool) { - for _, shard := range m.Shards { - if shard.Path == path { - return shard, true - } - } - return ShardEntry{}, false -} - -func equivalentManifest(a, b Manifest) bool { - if a.Format != b.Format || a.Encrypted != b.Encrypted || !sameStrings(a.Recipients, b.Recipients) || a.Counts != b.Counts || len(a.Shards) != len(b.Shards) { - return false - } - for i := range a.Shards { - left, right := a.Shards[i], b.Shards[i] - left.Bytes, right.Bytes = 0, 0 - if left != right { - return false - } - } - return true -} - -func normalizedStrings(values []string) []string { - seen := map[string]struct{}{} - out := make([]string, 0, len(values)) - for _, value := range values { - value = strings.TrimSpace(value) - if value == "" { - continue - } - if _, ok := seen[value]; ok { - continue - } - seen[value] = struct{}{} - out = append(out, value) - } - sort.Strings(out) - return out -} - -func sameStrings(a, b []string) bool { - a, b = normalizedStrings(a), normalizedStrings(b) - if len(a) != len(b) { - return false - } - for i := range a { - if a[i] != b[i] { - return false - } +func toCrawlkitManifest(manifest Manifest) ckbackup.Manifest { + return ckbackup.Manifest{ + Format: manifest.Format, + Encrypted: manifest.Encrypted, + Exported: manifest.Exported, + Recipients: manifest.Recipients, + Counts: map[string]int{ + "contacts": manifest.Counts.Contacts, + "chats": manifest.Counts.Chats, + "folders": manifest.Counts.Folders, + "folder_chats": manifest.Counts.FolderChats, + "groups": manifest.Counts.Groups, + "participants": manifest.Counts.Participants, + "topics": manifest.Counts.Topics, + "messages": manifest.Counts.Messages, + }, + Shards: manifest.Shards, } - return true } -func removeStaleShards(repo string, shards []ShardEntry) error { - keep := map[string]struct{}{} - for _, shard := range shards { - keep[filepath.Clean(filepath.Join(repo, filepath.FromSlash(shard.Path)))] = struct{}{} +func fromCrawlkitManifest(manifest ckbackup.Manifest) Manifest { + participants := manifest.Counts["participants"] + if participants == 0 { + participants = manifest.Counts["group_participants"] } - root := filepath.Join(repo, "data") - if _, err := os.Stat(root); os.IsNotExist(err) { - return nil - } - var stale []string - if err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { - if err != nil || d == nil || d.IsDir() { - return err - } - if !strings.HasSuffix(path, ".age") { - return nil - } - clean := filepath.Clean(path) - if _, ok := keep[clean]; ok { - return nil - } - stale = append(stale, clean) - return nil - }); err != nil { - return err - } - for _, path := range stale { - rel, err := filepath.Rel(root, path) - if err != nil || rel == "." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) || filepath.IsAbs(rel) { - return fmt.Errorf("stale shard path escapes backup root: %s", path) - } - if err := os.Remove(path); err != nil { - return err - } + return Manifest{ + Format: manifest.Format, + Encrypted: manifest.Encrypted, + Exported: manifest.Exported, + Recipients: manifest.Recipients, + Counts: Counts{ + Contacts: manifest.Counts["contacts"], + Chats: manifest.Counts["chats"], + Folders: manifest.Counts["folders"], + FolderChats: manifest.Counts["folder_chats"], + Groups: manifest.Counts["groups"], + Participants: participants, + Topics: manifest.Counts["topics"], + Messages: manifest.Counts["messages"], + }, + Shards: manifest.Shards, } - return nil } func writeBackupReadme(repo string) error { @@ -535,8 +381,11 @@ README.md manifest.json data/chats.jsonl.gz.age data/contacts.jsonl.gz.age +data/folders.jsonl.gz.age +data/folder_chats.jsonl.gz.age data/groups.jsonl.gz.age data/group_participants.jsonl.gz.age +data/topics.jsonl.gz.age data/messages/YYYY/MM.jsonl.gz.age ` + "```" + ` @@ -567,21 +416,28 @@ may still contain shards decryptable by the compromised key. ` + "```bash" + ` telecrawl backup push +telecrawl backup push --tag snapshot/before-migration ` + "```" + ` The command pulls/rebases this checkout, refreshes the local telecrawl archive according to the normal sync policy, writes encrypted shards, updates the manifest, commits, and pushes this repository. +Every changed backup is a Git commit. Optional tags name important checkpoints; +tag names are visible Git metadata and should not contain sensitive text. + ## Restore ` + "```bash" + ` telecrawl backup pull +telecrawl backup snapshots +telecrawl --db /tmp/telecrawl-history.db backup pull --ref snapshot/before-migration ` + "```" + ` ` + "`backup pull`" + ` decrypts every shard with the local age identity, verifies the manifest hashes, validates the snapshot, and imports it into the configured -telecrawl archive database. +telecrawl archive database. Historical refs are read directly from Git objects +without changing this checkout's current branch. ## Recovery diff --git a/internal/backup/backup_test.go b/internal/backup/backup_test.go index ea84890..64aae20 100644 --- a/internal/backup/backup_test.go +++ b/internal/backup/backup_test.go @@ -3,13 +3,17 @@ package backup import ( "bytes" "context" + "fmt" "os" + "os/exec" "path/filepath" "strings" "testing" "time" "filippo.io/age" + ckbackup "github.com/openclaw/crawlkit/backup" + "github.com/openclaw/crawlkit/mirror" "github.com/openclaw/telecrawl/internal/store" ) @@ -164,6 +168,107 @@ func TestEncryptedBackupPushPull(t *testing.T) { } } +func TestHistoricalSnapshotRestore(t *testing.T) { + ctx := context.Background() + now := time.Date(2026, 6, 19, 12, 0, 0, 0, time.UTC) + data := store.SnapshotData{ + Chats: []store.Chat{{JID: "chat", Kind: "dm", Name: "Chat", LastMessageAt: now}}, + Messages: []store.Message{{SourcePK: 1, ChatJID: "chat", MessageID: "one", Timestamp: now, Text: "first snapshot", RawType: 0}}, + } + st := openFixtureStore(t, "source.db") + if err := st.ImportSnapshot(ctx, data, "/fixture", now); err != nil { + t.Fatal(err) + } + remote := filepath.Join(t.TempDir(), "remote.git") + runGit(t, "", "init", "--bare", remote) + repo := filepath.Join(t.TempDir(), "backup") + identity := filepath.Join(t.TempDir(), "age.key") + configPath := filepath.Join(t.TempDir(), "backup.json") + if _, _, err := Init(ctx, Options{ConfigPath: configPath, Repo: repo, Remote: remote, Identity: identity, Push: false}); err != nil { + t.Fatal(err) + } + if _, err := Push(ctx, st, Options{ConfigPath: configPath, Push: false, Tag: "snapshot/initial"}); err != nil { + t.Fatal(err) + } + initial, err := resolveCommit(ctx, repo, "snapshot/initial") + if err != nil { + t.Fatal(err) + } + data.Messages = append(data.Messages, store.Message{SourcePK: 2, ChatJID: "chat", MessageID: "two", Timestamp: now.Add(time.Minute), Text: "second snapshot", RawType: 0}) + if err := st.ImportSnapshot(ctx, data, "/fixture", now.Add(time.Minute)); err != nil { + t.Fatal(err) + } + if _, err := Push(ctx, st, Options{ConfigPath: configPath, Push: true, Tag: "snapshot/current"}); err != nil { + t.Fatal(err) + } + current, err := resolveCommit(ctx, repo, "HEAD") + if err != nil { + t.Fatal(err) + } + snapshots, snapshotsRepo, err := Snapshots(ctx, Options{ConfigPath: configPath, Limit: 10}) + if err != nil { + t.Fatal(err) + } + if snapshotsRepo != repo || len(snapshots) != 2 || snapshots[0].Ref != current || snapshots[1].Ref != initial { + t.Fatalf("unexpected snapshots repo=%s snapshots=%+v", snapshotsRepo, snapshots) + } + restored := openFixtureStore(t, "restored.db") + pulled, err := Pull(ctx, restored, Options{ConfigPath: configPath, Ref: "snapshot/initial"}) + if err != nil { + t.Fatal(err) + } + if pulled.Ref != initial || pulled.Messages != 1 { + t.Fatalf("unexpected historical restore: %+v", pulled) + } + after, err := resolveCommit(ctx, repo, "HEAD") + if err != nil { + t.Fatal(err) + } + if after != current { + t.Fatalf("historical restore changed checkout from %s to %s", current, after) + } + if _, err := Push(ctx, st, Options{ConfigPath: configPath, Push: false, Tag: "snapshot/initial"}); err == nil { + t.Fatal("moving an immutable snapshot tag should fail") + } +} + +func TestEmptyBackupPreservesCountsAndIsIdempotent(t *testing.T) { + ctx := context.Background() + st := openFixtureStore(t, "empty.db") + repo := filepath.Join(t.TempDir(), "backup") + if err := os.MkdirAll(repo, 0o700); err != nil { + t.Fatal(err) + } + runGit(t, repo, "init") + identity := filepath.Join(t.TempDir(), "age.key") + recipient, err := EnsureIdentity(identity) + if err != nil { + t.Fatal(err) + } + opts := Options{Repo: repo, Identity: identity, Recipients: []string{recipient}, Push: false} + first, err := Push(ctx, st, opts) + if err != nil { + t.Fatal(err) + } + if !first.Changed { + t.Fatal("first empty backup should commit") + } + manifest, err := ckbackup.ReadManifest(repo) + if err != nil { + t.Fatal(err) + } + if messages, ok := manifest.Counts["messages"]; !ok || messages != 0 { + t.Fatalf("empty message count missing: %+v", manifest.Counts) + } + second, err := Push(ctx, st, opts) + if err != nil { + t.Fatal(err) + } + if second.Changed { + t.Fatal("identical empty backup should not commit") + } +} + func TestConfigRoundTrip(t *testing.T) { path := filepath.Join(t.TempDir(), "backup.json") cfg := DefaultConfig() @@ -216,28 +321,28 @@ func TestCryptoHelpers(t *testing.T) { if fromIdentity != recipient { t.Fatalf("recipient mismatch: %q != %q", fromIdentity, recipient) } - encrypted, hash, err := encryptShard([]byte("private text\n"), []string{recipient}) + encrypted, hash, err := ckbackup.EncryptShard([]byte("private text\n"), []string{recipient}) if err != nil { t.Fatal(err) } - if hash != sha256Hex([]byte("private text\n")) || strings.Contains(string(encrypted), "private text") { + if hash != ckbackup.SHA256Hex([]byte("private text\n")) || strings.Contains(string(encrypted), "private text") { t.Fatal("encrypted shard mismatch") } tmp := filepath.Join(t.TempDir(), "shard.age") if err := os.WriteFile(tmp, encrypted, 0o600); err != nil { t.Fatal(err) } - plain, err := decryptShard(encrypted, identity) + plain, err := ckbackup.DecryptShard(encrypted, identity) if err != nil { t.Fatal(err) } if string(plain) != "private text\n" { t.Fatalf("decrypt mismatch: %q", plain) } - if _, _, err := encryptShard([]byte("x"), []string{"bad"}); err == nil { + if _, _, err := ckbackup.EncryptShard([]byte("x"), []string{"bad"}); err == nil { t.Fatal("expected bad recipient error") } - if _, _, err := encryptShard([]byte("x"), nil); err == nil { + if _, _, err := ckbackup.EncryptShard([]byte("x"), nil); err == nil { t.Fatal("expected missing recipient encrypt error") } emptyIdentity := filepath.Join(t.TempDir(), "empty.key") @@ -260,14 +365,14 @@ func TestCryptoHelpers(t *testing.T) { if _, err := RecipientFromIdentity(badIdentity); err == nil { t.Fatal("expected bad identity parse error") } - if _, err := decryptShard([]byte("not age"), identity); err == nil { + if _, err := ckbackup.DecryptShard([]byte("not age"), identity); err == nil { t.Fatal("expected bad ciphertext error") } otherIdentity := filepath.Join(t.TempDir(), "other.key") if _, err := EnsureIdentity(otherIdentity); err != nil { t.Fatal(err) } - if _, err := decryptShard(encrypted, otherIdentity); err == nil { + if _, err := ckbackup.DecryptShard(encrypted, otherIdentity); err == nil { t.Fatal("expected wrong identity decrypt error") } recipientValue, err := age.ParseX25519Recipient(recipient) @@ -285,7 +390,7 @@ func TestCryptoHelpers(t *testing.T) { if err := w.Close(); err != nil { t.Fatal(err) } - if _, err := decryptShard(rawAge.Bytes(), identity); err == nil { + if _, err := ckbackup.DecryptShard(rawAge.Bytes(), identity); err == nil { t.Fatal("expected non-gzip decrypt error") } if _, err := EnsureIdentity(filepath.Join(t.TempDir(), "missing", "dir")); err != nil { @@ -294,17 +399,17 @@ func TestCryptoHelpers(t *testing.T) { } func TestSnapshotErrorAndUtilityPaths(t *testing.T) { - if _, _, err := encodeJSONL(1); err == nil { + if _, _, err := ckbackup.EncodeJSONL(1); err == nil { t.Fatal("expected unsupported JSONL row type") } var contacts []store.Contact - if err := decodeJSONL([]byte("{bad json}\n"), &contacts); err == nil { + if err := ckbackup.DecodeJSONL([]byte("{bad json}\n"), &contacts); err == nil { t.Fatal("expected invalid JSONL error") } - if err := removeStaleShards(t.TempDir(), nil); err != nil { + if err := ckbackup.RemoveStaleShards(t.TempDir(), nil); err != nil { t.Fatal(err) } - if equivalentManifest(Manifest{Format: 1}, Manifest{Format: 2}) { + if ckbackup.EquivalentManifest(toCrawlkitManifest(Manifest{Format: 1}), toCrawlkitManifest(Manifest{Format: 2})) { t.Fatal("different manifests should not be equivalent") } if _, err := readSnapshot(Config{}, Manifest{Format: 99}); err == nil { @@ -319,13 +424,13 @@ func TestSnapshotErrorAndUtilityPaths(t *testing.T) { t.Fatal(err) } repo := t.TempDir() - if _, err := resolveShardPath(repo, "../outside.age"); err == nil { + if _, err := ckbackup.ResolveShardPath(repo, "../outside.age"); err == nil { t.Fatal("expected escaping shard path error") } - if _, err := resolveShardPath(repo, "manifest.json"); err == nil { + if _, err := ckbackup.ResolveShardPath(repo, "manifest.json"); err == nil { t.Fatal("expected invalid shard path error") } - encrypted, hash, err := encryptShard([]byte("{}\n"), []string{recipient}) + encrypted, hash, err := ckbackup.EncryptShard([]byte("{}\n"), []string{recipient}) if err != nil { t.Fatal(err) } @@ -346,7 +451,7 @@ func TestSnapshotErrorAndUtilityPaths(t *testing.T) { if _, err := readSnapshot(cfg, badHashManifest); err == nil { t.Fatal("expected hash mismatch") } - duplicatePlain, duplicateHash, err := encryptShard([]byte(`{"source_pk":1,"chat_jid":"chat","message_id":"a","timestamp":"2026-04-27T12:00:00Z","raw_type":0}`+"\n"+`{"source_pk":1,"chat_jid":"chat","message_id":"b","timestamp":"2026-04-27T12:00:01Z","raw_type":0}`+"\n"), []string{recipient}) + duplicatePlain, duplicateHash, err := ckbackup.EncryptShard([]byte(`{"source_pk":1,"chat_jid":"chat","message_id":"a","timestamp":"2026-04-27T12:00:00Z","raw_type":0}`+"\n"+`{"source_pk":1,"chat_jid":"chat","message_id":"b","timestamp":"2026-04-27T12:00:01Z","raw_type":0}`+"\n"), []string{recipient}) if err != nil { t.Fatal(err) } @@ -363,7 +468,7 @@ func TestSnapshotErrorAndUtilityPaths(t *testing.T) { if err := duplicateData.Validate(); err == nil { t.Fatal("expected duplicate restored data validation error") } - if err := writeManifest(repo, Manifest{Format: formatVersion}); err != nil { + if err := ckbackup.WriteManifest(repo, toCrawlkitManifest(Manifest{Format: formatVersion})); err != nil { t.Fatal(err) } if _, err := readManifest(repo); err != nil { @@ -381,7 +486,7 @@ func TestSnapshotErrorAndUtilityPaths(t *testing.T) { if err := os.WriteFile(stalePath, []byte("stale"), 0o600); err != nil { t.Fatal(err) } - if err := removeStaleShards(repo, []ShardEntry{{Path: filepath.ToSlash(shardPath)}}); err != nil { + if err := ckbackup.RemoveStaleShards(repo, []ShardEntry{{Path: filepath.ToSlash(shardPath)}}); err != nil { t.Fatal(err) } if _, err := os.Stat(stalePath); !os.IsNotExist(err) { @@ -405,6 +510,25 @@ func TestGitHelpersWithoutRemote(t *testing.T) { } } +func TestEnsureRepoFallsBackToLocalInitWhenCloneFails(t *testing.T) { + ctx := context.Background() + repo := filepath.Join(t.TempDir(), "backup") + remote := filepath.Join(t.TempDir(), "missing.git") + if err := ensureRepo(ctx, Config{Repo: repo, Remote: remote}); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(repo, ".git")); err != nil { + t.Fatal(err) + } + out, err := gitOutput(ctx, repo, "remote", "get-url", "origin") + if err != nil { + t.Fatal(err) + } + if strings.TrimSpace(string(out)) != remote { + t.Fatalf("origin = %q, want %q", strings.TrimSpace(string(out)), remote) + } +} + func TestTopLevelErrorPaths(t *testing.T) { ctx := context.Background() source := openFixtureStore(t, "source.db") @@ -472,7 +596,27 @@ func openFixtureStore(t *testing.T, name string) *store.Store { func runGit(t *testing.T, dir string, args ...string) { t.Helper() - if err := git(context.Background(), dir, args...); err != nil { + if _, err := gitOutput(context.Background(), dir, args...); err != nil { t.Fatal(err) } } + +func resolveCommit(ctx context.Context, repo, ref string) (string, error) { + return mirror.ResolveCommit(ctx, mirror.Options{RepoPath: repo, Branch: "main"}, ref) +} + +func gitOutput(ctx context.Context, dir string, args ...string) ([]byte, error) { + cmd := exec.CommandContext(ctx, "git", args...) // #nosec G204 -- tests pass only fixed Git commands and temporary paths. + cmd.Dir = dir + cmd.Env = append(os.Environ(), + "GIT_AUTHOR_NAME=telecrawl-test", + "GIT_AUTHOR_EMAIL=telecrawl-test@example.invalid", + "GIT_COMMITTER_NAME=telecrawl-test", + "GIT_COMMITTER_EMAIL=telecrawl-test@example.invalid", + ) + out, err := cmd.CombinedOutput() + if err != nil { + return nil, fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(string(out))) + } + return out, nil +} diff --git a/internal/backup/config.go b/internal/backup/config.go index 0fe51ed..dafe8fa 100644 --- a/internal/backup/config.go +++ b/internal/backup/config.go @@ -27,6 +27,9 @@ type Options struct { Identity string Recipients []string Push bool + Ref string + Tag string + Limit int } func DefaultConfig() Config { diff --git a/internal/backup/crypto.go b/internal/backup/crypto.go index a12ac88..eda6287 100644 --- a/internal/backup/crypto.go +++ b/internal/backup/crypto.go @@ -9,15 +9,3 @@ func EnsureIdentity(path string) (string, error) { func RecipientFromIdentity(path string) (string, error) { return ckbackup.RecipientFromIdentity(path) } - -func encryptShard(plaintext []byte, recipientStrings []string) ([]byte, string, error) { - return ckbackup.EncryptShard(plaintext, recipientStrings) -} - -func decryptShard(ciphertext []byte, identityPath string) ([]byte, error) { - return ckbackup.DecryptShard(ciphertext, identityPath) -} - -func sha256Hex(data []byte) string { - return ckbackup.SHA256Hex(data) -} diff --git a/internal/backup/git.go b/internal/backup/git.go index 31f3689..b1e62df 100644 --- a/internal/backup/git.go +++ b/internal/backup/git.go @@ -1,91 +1,61 @@ package backup import ( - "bytes" "context" "fmt" "os" - "os/exec" "path/filepath" "strings" + + "github.com/openclaw/crawlkit/mirror" ) +func mirrorOptions(cfg Config) mirror.Options { + return mirror.Options{RepoPath: cfg.Repo, Remote: cfg.Remote, Branch: "main"} +} + +func syncOptions(cfg Config) mirror.Options { + opts := mirrorOptions(cfg) + if _, err := os.Stat(filepath.Join(cfg.Repo, ".git")); err == nil { + opts.Remote = "" + } + return opts +} + func ensureRepo(ctx context.Context, cfg Config) error { if strings.TrimSpace(cfg.Repo) == "" { return fmt.Errorf("backup repo path is required") } - if _, err := os.Stat(filepath.Join(cfg.Repo, ".git")); err == nil { - pullErr := git(ctx, cfg.Repo, "pull", "--rebase") - if pullErr != nil { - hasHead := git(ctx, cfg.Repo, "rev-parse", "--verify", "HEAD") == nil - if !hasHead { - return nil - } - if strings.Contains(pullErr.Error(), "no tracking information") || - strings.Contains(pullErr.Error(), "No remote repository specified") || - strings.Contains(pullErr.Error(), "no such ref was fetched") { - return nil - } - return pullErr - } - return nil - } - if strings.TrimSpace(cfg.Remote) != "" { - if err := os.MkdirAll(filepath.Dir(cfg.Repo), 0o700); err != nil { - return err - } - if err := git(ctx, "", "clone", cfg.Remote, cfg.Repo); err == nil { - return nil - } - } - if err := os.MkdirAll(cfg.Repo, 0o700); err != nil { + opts := syncOptions(cfg) + err := mirror.SyncCurrentForWrite(ctx, opts) + if err == nil || opts.Remote == "" { return err } - if err := git(ctx, cfg.Repo, "init"); err != nil { + if _, statErr := os.Stat(filepath.Join(cfg.Repo, ".git")); statErr == nil { return err } - if strings.TrimSpace(cfg.Remote) != "" { - if err := git(ctx, cfg.Repo, "remote", "add", "origin", cfg.Remote); err != nil { - return err - } + local := opts + local.Remote = "" + if initErr := mirror.EnsureRepo(ctx, local); initErr != nil { + return fmt.Errorf("initialize backup repo after clone failed: %w", initErr) + } + if remoteErr := mirror.EnsureRemote(ctx, opts); remoteErr != nil { + return fmt.Errorf("configure backup remote after clone failed: %w", remoteErr) } return nil } -func commitAndPush(ctx context.Context, cfg Config, message string, push bool) (bool, error) { - if err := git(ctx, cfg.Repo, "add", "."); err != nil { - return false, err - } - if err := git(ctx, cfg.Repo, "diff", "--cached", "--quiet"); err == nil { - return false, nil - } - if err := git(ctx, cfg.Repo, "commit", "-m", message); err != nil { - return false, err - } - if push { - if err := git(ctx, cfg.Repo, "push", "-u", "origin", "HEAD"); err != nil { - return true, err - } +func ensureRepoForRead(ctx context.Context, cfg Config) error { + if strings.TrimSpace(cfg.Repo) == "" { + return fmt.Errorf("backup repo path is required") } - return true, nil + return mirror.Fetch(ctx, syncOptions(cfg)) } -func git(ctx context.Context, dir string, args ...string) error { - cmd := exec.CommandContext(ctx, "git", args...) // #nosec G204 -- telecrawl only passes fixed git subcommands plus configured repo paths. - cmd.Dir = dir - cmd.Env = append(os.Environ(), - "GIT_AUTHOR_NAME=telecrawl", - "GIT_AUTHOR_EMAIL=telecrawl@example.invalid", - "GIT_COMMITTER_NAME=telecrawl", - "GIT_COMMITTER_EMAIL=telecrawl@example.invalid", - ) - var stderr bytes.Buffer - cmd.Stderr = &stderr - if err := cmd.Run(); err != nil { - if stderr.Len() > 0 { - return fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String())) - } - return fmt.Errorf("git %s: %w", strings.Join(args, " "), err) +func commitAndPush(ctx context.Context, cfg Config, message string, push bool) (bool, error) { + changed, err := mirror.Commit(ctx, mirrorOptions(cfg), message) + if err != nil || !push || !changed { + return changed, err } - return nil + return true, mirror.PushAtomic(ctx, mirrorOptions(cfg), "HEAD") } diff --git a/internal/backup/history.go b/internal/backup/history.go new file mode 100644 index 0000000..b8bde79 --- /dev/null +++ b/internal/backup/history.go @@ -0,0 +1,69 @@ +package backup + +import ( + "context" + "fmt" + "time" + + ckbackup "github.com/openclaw/crawlkit/backup" + "github.com/openclaw/crawlkit/mirror" + "github.com/openclaw/telecrawl/internal/store" +) + +const defaultSnapshotLimit = ckbackup.DefaultHistoryLimit + +type Snapshot struct { + Ref string `json:"ref"` + Tags []string `json:"tags,omitempty"` + Exported time.Time `json:"exported"` + Counts Counts `json:"counts"` + Shards int `json:"shards"` +} + +func Snapshots(ctx context.Context, opts Options) ([]Snapshot, string, error) { + cfg, err := ResolveOptions(opts) + if err != nil { + return nil, "", err + } + limit := opts.Limit + if limit == 0 { + limit = defaultSnapshotLimit + } + if limit < 1 { + return nil, "", fmt.Errorf("snapshot limit must be greater than zero") + } + history, err := ckbackup.History(ctx, syncOptions(cfg), limit) + if err != nil { + return nil, "", err + } + out := make([]Snapshot, 0, len(history)) + for _, entry := range history { + manifest := fromCrawlkitManifest(entry.Manifest) + out = append(out, Snapshot{Ref: entry.Ref, Tags: entry.Tags, Exported: manifest.Exported, Counts: manifest.Counts, Shards: len(manifest.Shards)}) + } + return out, cfg.Repo, nil +} + +func readManifestAtRef(ctx context.Context, repo, requested string) (Manifest, string, error) { + manifest, commit, err := ckbackup.ReadManifestAt(ctx, mirror.Options{RepoPath: repo, Branch: "main"}, requested) + if err != nil { + return Manifest{}, "", err + } + return fromCrawlkitManifest(manifest), commit, nil +} + +func readSnapshotAtRef(ctx context.Context, cfg Config, manifest Manifest, commit string) (store.SnapshotData, error) { + shards, _, err := ckbackup.ReadSnapshotAt(ctx, crawlkitConfig(cfg), mirrorOptions(cfg), toCrawlkitManifest(manifest), commit) + if err != nil { + return store.SnapshotData{}, err + } + return decodeSnapshot(shards) +} + +func validateSnapshotTag(ctx context.Context, repo, requested string) error { + return mirror.ValidateTag(ctx, mirror.Options{RepoPath: repo, Branch: "main"}, requested) +} + +func tagSnapshot(ctx context.Context, cfg Config, requested string) (string, error) { + return mirror.CreateImmutableTag(ctx, mirrorOptions(cfg), requested) +} diff --git a/internal/cli/cli.go b/internal/cli/cli.go index e14b019..73f0986 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -14,6 +14,7 @@ import ( "time" "unicode" + "github.com/openclaw/crawlkit/control" "github.com/openclaw/telecrawl/internal/backup" "github.com/openclaw/telecrawl/internal/store" "github.com/openclaw/telecrawl/internal/telegramdesktop" @@ -428,15 +429,6 @@ func (r *runtime) runContacts(args []string) error { }) } -type contactExport struct { - Contacts []exportedContact `json:"contacts"` -} - -type exportedContact struct { - DisplayName string `json:"display_name"` - PhoneNumbers []string `json:"phone_numbers"` -} - func (r *runtime) runContactsExport(args []string) error { fs := flag.NewFlagSet("telecrawl contacts export", flag.ContinueOnError) fs.SetOutput(io.Discard) @@ -451,12 +443,16 @@ func (r *runtime) runContactsExport(args []string) error { if err != nil { return err } - return r.print(contactExport{Contacts: exportContacts(contacts)}) + export := control.ContactExport{Contacts: exportContacts(contacts)} + if err := control.ValidateContactExport(export); err != nil { + return err + } + return r.print(export) }) } -func exportContacts(contacts []store.Contact) []exportedContact { - out := make([]exportedContact, 0, len(contacts)) +func exportContacts(contacts []store.Contact) []control.Contact { + out := make([]control.Contact, 0, len(contacts)) byPhone := map[string]store.Contact{} phoneOrder := make([]string, 0, len(contacts)) for _, contact := range contacts { @@ -480,7 +476,7 @@ func exportContacts(contacts []store.Contact) []exportedContact { for _, phone := range phoneOrder { contact := byPhone[phone] name := contactDisplayName(contact) - out = append(out, exportedContact{DisplayName: name, PhoneNumbers: []string{phone}}) + out = append(out, control.Contact{DisplayName: name, PhoneNumbers: []string{phone}}) } return out } @@ -657,7 +653,7 @@ func (r *runtime) messageFilter(name string, args []string, requireQuery bool) ( func (r *runtime) runBackup(args []string) error { if len(args) == 0 { - return usageErr(errors.New("backup needs subcommand: init, push, pull, status")) + return usageErr(errors.New("backup needs subcommand: init, push, pull, status, snapshots")) } switch args[0] { case "init": @@ -668,6 +664,8 @@ func (r *runtime) runBackup(args []string) error { return r.backupPull(args[1:]) case "status": return r.backupStatus(args[1:]) + case "snapshots": + return r.backupSnapshots(args[1:]) default: return usageErr(fmt.Errorf("unknown backup command %q", args[0])) } @@ -681,6 +679,9 @@ func backupFlags(name string) (*flag.FlagSet, *backup.Options, *bool) { fs.StringVar(&opts.Repo, "repo", "", "") fs.StringVar(&opts.Remote, "remote", "", "") fs.StringVar(&opts.Identity, "identity", "", "") + fs.StringVar(&opts.Ref, "ref", "", "") + fs.StringVar(&opts.Tag, "tag", "", "") + fs.IntVar(&opts.Limit, "limit", 20, "") fs.Func("recipient", "", func(value string) error { opts.Recipients = append(opts.Recipients, value) return nil @@ -743,6 +744,27 @@ func (r *runtime) backupStatus(args []string) error { return r.print(map[string]any{"repo": repo, "manifest": manifest}) } +func (r *runtime) backupSnapshots(args []string) error { + fs, opts, _ := backupFlags("telecrawl backup snapshots") + if err := fs.Parse(args); err != nil { + return usageErr(err) + } + if fs.NArg() != 0 { + return usageErr(errors.New("backup snapshots takes flags only")) + } + if opts.Limit < 1 { + return usageErr(errors.New("backup snapshots --limit must be greater than zero")) + } + snapshots, repo, err := backup.Snapshots(r.ctx, *opts) + if err != nil { + return err + } + if r.json { + return r.print(map[string]any{"repo": repo, "snapshots": snapshots}) + } + return r.print(snapshots) +} + func (r *runtime) printProbe(report telegramdesktop.Report) error { if r.json { enc := json.NewEncoder(r.stdout) @@ -853,6 +875,17 @@ func (r *runtime) print(v any) error { } } return nil + case []backup.Snapshot: + for _, snapshot := range value { + ref := snapshot.Ref + if len(snapshot.Tags) > 0 { + ref = snapshot.Tags[0] + } + if _, err := fmt.Fprintf(r.stdout, "%s\t%s\t%d\t%d\t%s\n", ref, snapshot.Exported.Format(time.RFC3339), snapshot.Counts.Messages, snapshot.Shards, strings.Join(snapshot.Tags, ",")); err != nil { + return err + } + } + return nil default: enc.SetIndent("", " ") return enc.Encode(v) @@ -888,7 +921,7 @@ usage: telecrawl [--json] topics --chat ID [--limit N] telecrawl [--json] messages [--chat ID] [--topic ID] [--limit N] [--after DATE] telecrawl [--json] search "query" [--chat ID] [--topic ID] - telecrawl [--json] backup init|push|pull|status + telecrawl [--json] backup init|push|pull|status|snapshots telecrawl version notes: diff --git a/internal/store/store.go b/internal/store/store.go index d5e79e5..7b390ef 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -10,6 +10,7 @@ import ( "strings" "time" + ckstore "github.com/openclaw/crawlkit/store" _ "modernc.org/sqlite" ) @@ -533,8 +534,12 @@ func (s *Store) messages(ctx context.Context, filter MessageFilter, search bool) args := []any{} prefix := "" if search { + ftsQuery, err := ckstore.FTS5Terms(filter.Query, "") + if err != nil { + return nil, err + } query = `select m.source_pk,m.chat_jid,coalesce(m.chat_name,''),m.msg_id,coalesce(m.sender_jid,''),coalesce(m.sender_name,''),m.ts,coalesce(m.edit_ts,0),m.from_me,coalesce(m.text,''),m.raw_type,coalesce(m.message_type,''),coalesce(m.media_type,''),coalesce(m.media_title,''),coalesce(m.media_path,''),coalesce(m.media_url,''),coalesce(m.media_size,0),coalesce(m.metadata_type,''),coalesce(m.metadata_title,''),coalesce(m.metadata_url,''),coalesce(m.metadata_json,''),m.starred,coalesce(m.topic_id,''),coalesce(m.reply_to_msg_id,''),coalesce(m.reply_to_chat_jid,''),coalesce(m.thread_id,''),coalesce(m.forward_json,''),coalesce(m.reactions_json,''),coalesce(m.views,0),coalesce(m.forwards,0),coalesce(m.replies_count,0),coalesce(m.pinned,0),snippet(messages_fts,0,'[',']','...',12) from messages_fts f join messages m on m.rowid=f.rowid where messages_fts match ?` - args = append(args, filter.Query) + args = append(args, ftsQuery) prefix = "m." } if filter.ChatJID != "" { From cea247977b0f6d8366bd4eea1ea4281eb6990a09 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 19 Jun 2026 06:51:30 -0400 Subject: [PATCH 2/2] build(deps): update Go dependencies --- go.mod | 49 +++++----- go.sum | 110 ++++++++++++---------- internal/telegramdesktop/importer_test.go | 6 +- internal/telegramdesktop/tdata.go | 2 +- 4 files changed, 96 insertions(+), 71 deletions(-) diff --git a/go.mod b/go.mod index 505e672..7611356 100644 --- a/go.mod +++ b/go.mod @@ -4,14 +4,21 @@ go 1.26.4 require ( filippo.io/age v1.3.1 - modernc.org/sqlite v1.50.1 + modernc.org/sqlite v1.52.0 +) + +require ( + github.com/andybalholm/brotli v1.2.1 // indirect + github.com/gotd/log v0.1.0 // indirect + github.com/refraction-networking/utls v1.8.2 // indirect + github.com/yuin/goldmark v1.8.2 // indirect ) require ( github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/coder/websocket v1.8.14 // indirect - github.com/dlclark/regexp2 v1.11.5 // indirect + github.com/coder/websocket v1.8.15 // indirect + github.com/dlclark/regexp2 v1.12.0 // indirect github.com/fatih/color v1.19.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-faster/errors v0.7.1 // indirect @@ -20,24 +27,24 @@ require ( github.com/go-faster/yaml v0.4.6 // indirect github.com/gotd/ige v0.2.2 // indirect github.com/gotd/neo v0.1.5 // indirect - github.com/gotd/td v0.145.1 - github.com/klauspost/compress v1.18.5 // indirect - github.com/mattn/go-colorable v0.1.14 // indirect - github.com/ogen-go/ogen v1.20.3 // indirect + github.com/gotd/td v0.159.0 + github.com/klauspost/compress v1.18.6 // indirect + github.com/mattn/go-colorable v0.1.15 // indirect + github.com/ogen-go/ogen v1.22.0 // indirect github.com/segmentio/asm v1.2.1 // indirect github.com/shopspring/decimal v1.4.0 // indirect - go.opentelemetry.io/otel v1.42.0 // indirect - go.opentelemetry.io/otel/metric v1.42.0 // indirect - go.opentelemetry.io/otel/trace v1.42.0 // indirect + go.opentelemetry.io/otel v1.44.0 // indirect + go.opentelemetry.io/otel/metric v1.44.0 // indirect + go.opentelemetry.io/otel/trace v1.44.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.27.1 // indirect - golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 // indirect - golang.org/x/mod v0.36.0 // indirect - golang.org/x/net v0.54.0 // indirect - golang.org/x/sync v0.20.0 // indirect - golang.org/x/text v0.37.0 // indirect - golang.org/x/tools v0.45.0 // indirect + go.uber.org/zap v1.28.0 // indirect + golang.org/x/exp v0.0.0-20260611194520-c48552f49976 // indirect + golang.org/x/mod v0.37.0 // indirect + golang.org/x/net v0.56.0 // indirect + golang.org/x/sync v0.21.0 // indirect + golang.org/x/text v0.38.0 // indirect + golang.org/x/tools v0.46.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect rsc.io/qr v0.2.0 // indirect ) @@ -48,11 +55,11 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/mattn/go-isatty v0.0.22 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect - github.com/openclaw/crawlkit v0.7.0 + github.com/openclaw/crawlkit v0.12.3-0.20260619102715-6e14735bb248 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - golang.org/x/crypto v0.52.0 - golang.org/x/sys v0.45.0 // indirect - modernc.org/libc v1.72.3 // indirect + golang.org/x/crypto v0.53.0 + golang.org/x/sys v0.46.0 // indirect + modernc.org/libc v1.73.4 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect ) diff --git a/go.sum b/go.sum index 50d7774..73d8346 100644 --- a/go.sum +++ b/go.sum @@ -4,16 +4,18 @@ filippo.io/age v1.3.1 h1:hbzdQOJkuaMEpRCLSN1/C5DX74RPcNCk6oqhKMXmZi0= filippo.io/age v1.3.1/go.mod h1:EZorDTYUxt836i3zdori5IJX/v2Lj6kWFU0cfh6C0D4= filippo.io/hpke v0.4.0 h1:p575VVQ6ted4pL+it6M00V/f2qTZITO0zgmdKCkd5+A= filippo.io/hpke v0.4.0/go.mod h1:EmAN849/P3qdeK+PCMkDpDm83vRHM5cDipBJ8xbQLVY= +github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro= +github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/coder/websocket v1.8.14 h1:9L0p0iKiNOibykf283eHkKUHHrpG7f65OE3BhhO7v9g= -github.com/coder/websocket v1.8.14/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg= +github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNUA= +github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ= -github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/dlclark/regexp2 v1.12.0 h1:0j4c5qQmnC6XOWNjP3PIXURXN2gWx76rd3KvgdPkCz8= +github.com/dlclark/regexp2 v1.12.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w= @@ -41,30 +43,36 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gotd/ige v0.2.2 h1:XQ9dJZwBfDnOGSTxKXBGP4gMud3Qku2ekScRjDWWfEk= github.com/gotd/ige v0.2.2/go.mod h1:tuCRb+Y5Y3eNTo3ypIfNpQ4MFjrnONiL2jN2AKZXmb0= +github.com/gotd/log v0.1.0 h1:4LJUEvafD1xtBwx2QkrlzFnRgbYXTlWqJPDi8BvrLbU= +github.com/gotd/log v0.1.0/go.mod h1:5ilhdu1Ux0QvDY/FF3Ojfw24Ws3SlCtyLwOpXy8KYXs= +github.com/gotd/log/logzap v0.1.1 h1:O6l7d8HUbODe+UMcrM47eXYDwdJ6RNmpQejLjrlcEIQ= +github.com/gotd/log/logzap v0.1.1/go.mod h1:5ObZkITbfhbsBOLzBkzmMk9QxXc0eNQpimau7zRL+Y8= github.com/gotd/neo v0.1.5 h1:oj0iQfMbGClP8xI59x7fE/uHoTJD7NZH9oV1WNuPukQ= github.com/gotd/neo v0.1.5/go.mod h1:9A2a4bn9zL6FADufBdt7tZt+WMhvZoc5gWXihOPoiBQ= -github.com/gotd/td v0.145.1 h1:ralLU/3r7qo+pc4D7cSGIWpe3RAm50BwtVGdru6aGwQ= -github.com/gotd/td v0.145.1/go.mod h1:azq4M4Z4BfNMeOmruLldbgHZAbfaW5P1JtcSB33hGf4= +github.com/gotd/td v0.159.0 h1:kKXt2NLmfIOgebbFS34FSlZbdydaf5fsta+nP69nP+w= +github.com/gotd/td v0.159.0/go.mod h1:rdZ2NfOMUViApJa3EvYJ94GAxENjCB0b98tJbfS9NCc= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= -github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= -github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= +github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= -github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY= +github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4= github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= -github.com/ogen-go/ogen v1.20.3 h1:1tvJuJE0BnQ7Nukd6ykiTOP0ucfL0yrAjHUg3S1DCQk= -github.com/ogen-go/ogen v1.20.3/go.mod h1:sJ1pJVp4S1RcSZlYIiMLo0QSMSt2pls4zfrc+hNKnzk= -github.com/openclaw/crawlkit v0.7.0 h1:mh8ZDEUP5hOQZcIY2DVo6Chy1SGu/2Re4olCuT/578k= -github.com/openclaw/crawlkit v0.7.0/go.mod h1:2XkSx3N8yzyjc+Jyf1Zl9VEQVlElh/dzBMW1cRMQQGw= +github.com/ogen-go/ogen v1.22.0 h1:7wU+jcIKg/JBAhM95909ULLdAkGr43KQOuvNpJ7Mxb4= +github.com/ogen-go/ogen v1.22.0/go.mod h1:7BOh9a51QiPCC92RMrj1LlkLjejhBAyPhR+oMc6lR9g= +github.com/openclaw/crawlkit v0.12.3-0.20260619102715-6e14735bb248 h1:EpOaEZl2IDx4Ck1UoULzMyENijI80Qn98WvsHztARPo= +github.com/openclaw/crawlkit v0.12.3-0.20260619102715-6e14735bb248/go.mod h1:GwfF/ZPPIaAy1lKcWIhA/YgcOXPHA+8rJm5Ned+AuIc= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/refraction-networking/utls v1.8.2 h1:j4Q1gJj0xngdeH+Ox/qND11aEfhpgoEvV+S9iJ2IdQo= +github.com/refraction-networking/utls v1.8.2/go.mod h1:jkSOEkLqn+S/jtpEHPOsVv/4V4EVnelwbMQl4vCWXAM= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/segmentio/asm v1.2.1 h1:DTNbBqs57ioxAD4PrArqftgypG4/qNpXoJx8TVXxPR0= @@ -73,39 +81,45 @@ github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +github.com/yuin/goldmark v1.8.2 h1:kEGpgqJXdgbkhcOgBxkC0X0PmoPG1ZyoZ117rDVp4zE= +github.com/yuin/goldmark v1.8.2/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho= -go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc= -go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4= -go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI= -go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY= -go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= -go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= -golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= -golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 h1:Di6/M8l0O2lCLc6VVRWhgCiApHV8MnQurBnFSHsQtNY= -golang.org/x/exp v0.0.0-20230725093048-515e97ebf090/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= -golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= -golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= -golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w= -golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ= +go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo= +go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= +golang.org/x/exp v0.0.0-20260611194520-c48552f49976 h1:X8Hz2ImujgbmetVuW+w2YkyZChE3cBpZi2P158rTG9M= +golang.org/x/exp v0.0.0-20260611194520-c48552f49976/go.mod h1:vnf4pv9iKZXY58sQE1L86zmNWJ4159e1RkcWiLCkeEY= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= -golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= -golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= -golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= -golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= +golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +golang.org/x/tools v0.46.0 h1:7jTurBkPZu4moS/Uy4OQT1M+QBlsj3wejyZwsT8Z7rk= +golang.org/x/tools v0.46.0/go.mod h1:FrD85F8l+NWL+9XWBSyVSHO6Ne4jutsfIFba7AWQ5Ys= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -113,20 +127,20 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -modernc.org/cc/v4 v4.28.2 h1:3tQ0lf2ADtoby2EtSP+J7IE2SHwEJdP8ioR59wx7XpY= -modernc.org/cc/v4 v4.28.2/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= -modernc.org/ccgo/v4 v4.34.0 h1:yRLPFZieg532OT4rp4JFNIVcquwalMX26G95WQDqwCQ= -modernc.org/ccgo/v4 v4.34.0/go.mod h1:AS5WYMyBakQ+fhsHhtP8mWB82KTGPkNNJDGfGQCe0/A= +modernc.org/cc/v4 v4.28.4 h1:Hd/4Es+MBj+/7hSdZaisNyu6bv3V0Dp2MdllyfqaH+c= +modernc.org/cc/v4 v4.28.4/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.34.4 h1:OVnSOWQjVKOYkFxoHYB+qQmSHK5gqMqARM+K9DpR/Ws= +modernc.org/ccgo/v4 v4.34.4/go.mod h1:qdKqE8FNIYyysougB1RX9MxCzp5oJOcQXSobANJ4TuE= modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= -modernc.org/gc/v3 v3.1.2 h1:ZtDCnhonXSZexk/AYsegNRV1lJGgaNZJuKjJSWKyEqo= -modernc.org/gc/v3 v3.1.2/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/gc/v3 v3.1.3 h1:6QAplYyVO+KdPW3pGnqmJDUxtkec8ooEWvks/hhU3lc= +modernc.org/gc/v3 v3.1.3/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= -modernc.org/libc v1.72.3 h1:ZnDF4tXn4NBXFutMMQC4vtbTFSXhhKzR73fv0beZEAU= -modernc.org/libc v1.72.3/go.mod h1:dn0dZNnnn1clLyvRxLxYExxiKRZIRENOfqQ8XEeg4Qs= +modernc.org/libc v1.73.4 h1:+ra4Ui8ngyt8HDcO1FTDPWlkAh6yOdaO2yAoh8MddQA= +modernc.org/libc v1.73.4/go.mod h1:DXZ3eO8qMCNn2SnmTNCiC71nJ9Rcq3PsnpU6Vc4rWK8= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= @@ -135,8 +149,8 @@ modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= -modernc.org/sqlite v1.50.1 h1:l+cQvn0sd0zJJtfygGHuQJ5AjlrwXmWPw4KP3ZMwr9w= -modernc.org/sqlite v1.50.1/go.mod h1:tcNzv5p84E0skkmJn038y+hWJbLQXQqEnQfeh5r2JLM= +modernc.org/sqlite v1.52.0 h1:p4dhYh2tXZCiyaqHwRVJDjIGKWyXayiQpThxgDzJaxo= +modernc.org/sqlite v1.52.0/go.mod h1:tcNzv5p84E0skkmJn038y+hWJbLQXQqEnQfeh5r2JLM= modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= diff --git a/internal/telegramdesktop/importer_test.go b/internal/telegramdesktop/importer_test.go index 57585ef..9c15121 100644 --- a/internal/telegramdesktop/importer_test.go +++ b/internal/telegramdesktop/importer_test.go @@ -306,9 +306,13 @@ func TestTDataWebpageMediaFileFallback(t *testing.T) { if !ok || docFile.Name != "preview.pdf" || docFile.MIMEType != "application/pdf" { t.Fatalf("webpage document file = %#v ok=%v", docFile, ok) } - if _, ok := docFile.Location.(*tg.InputDocumentFileLocation); !ok { + docLocation, ok := docFile.Location.(*tg.InputDocumentFileLocation) + if !ok { t.Fatalf("webpage document location = %T", docFile.Location) } + if docLocation.ThumbSize != "" { + t.Fatalf("full webpage document thumb size = %q, want empty", docLocation.ThumbSize) + } photoPage := &tg.WebPage{} photoPage.SetPhoto(&tg.Photo{ diff --git a/internal/telegramdesktop/tdata.go b/internal/telegramdesktop/tdata.go index 1a372a3..26595e3 100644 --- a/internal/telegramdesktop/tdata.go +++ b/internal/telegramdesktop/tdata.go @@ -462,7 +462,7 @@ func telegramMessageFile(elem querymessages.Elem) (querymessages.File, bool) { return querymessages.File{ Name: firstNonEmpty(tdataDocumentFilename(doc), tdataDocumentAudioTitle(doc), fmt.Sprintf("doc%d", doc.ID)), MIMEType: doc.MimeType, - Location: doc.AsInputDocumentFileLocation(), + Location: doc.AsInputDocumentFileLocation(""), }, true } }