diff --git a/docs/imports.md b/docs/imports.md index 081170b..0bd8588 100644 --- a/docs/imports.md +++ b/docs/imports.md @@ -1,3 +1,7 @@ +--- +written_by: ai +--- + # Imports Imports project an external contact graph into the same markdown shape @@ -101,6 +105,54 @@ Same shape as birdclaw, but reads from [discrawl](https://github.com/steipete/discrawl)'s SQLite cache. Discord handles land under `accounts.discord`. +## Crawler Contacts + +```bash +clawdex import contacts --from telecrawl --dry-run +clawdex import contacts --from wacrawl --dry-run +clawdex import contacts --from /path/to/crawler --dry-run +``` + +Reads a local crawler's crawlkit metadata and runs its advertised +`contact-export` command. This is the v0 machine contract for source crawler +contacts: + +- metadata schema is `crawlkit.control.v1` +- command name is `contact-export` +- command is read-only and advertises `json: true` +- advertised `argv` includes `--json` plus any source-safe flags +- payload root is `contacts` +- each contact has only `display_name` and `phone_numbers` + +Source crawlers own source-native extraction and privacy filtering. Clawdex +owns canonical people, markdown storage, matching, and human edits. + +Crawler contact imports match existing people by source accounts, external IDs, +emails, or normalized phone numbers. They do not automatically merge by name +alone; a matching display name without a matching phone is treated as a new +person for now instead of risking a bad join. + +If one exported crawler contact contains a phone already owned by a different +person, clawdex leaves that conflicting phone off the matched person instead of +creating an automatic cross-person join. + +When a crawler contact matches an existing person, clawdex records that source +under the person's local markdown frontmatter: + +```yaml +sources: + telecrawl: + names: ["Ada Example"] + phones: ["15550100"] + wacrawl: + names: ["Ada Example"] + phones: ["+1 555 0100"] +``` + +That source evidence is local-only and stable across repeated imports. It lets +clawdex answer that a person was seen in Telegram or WhatsApp even when the +incoming phone number was already present and no canonical phone field changed. + ## Sync (preview-only) ```bash diff --git a/go.mod b/go.mod index feda2db..099f99a 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/openclaw/clawdex -go 1.26.3 +go 1.26.4 require ( github.com/alecthomas/kong v1.15.0 diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 21b20f5..55dc1e3 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -1,6 +1,7 @@ package cli import ( + "bytes" "context" "encoding/json" "errors" @@ -8,6 +9,8 @@ import ( "io" "os" "os/exec" + "path/filepath" + "slices" "sort" "strings" "time" @@ -16,6 +19,7 @@ import ( "github.com/openclaw/clawdex/internal/apple" "github.com/openclaw/clawdex/internal/avatar" "github.com/openclaw/clawdex/internal/birdclaw" + "github.com/openclaw/clawdex/internal/contactexport" "github.com/openclaw/clawdex/internal/discrawl" "github.com/openclaw/clawdex/internal/google" "github.com/openclaw/clawdex/internal/index" @@ -23,6 +27,7 @@ import ( "github.com/openclaw/clawdex/internal/model" "github.com/openclaw/clawdex/internal/repo" "github.com/openclaw/clawdex/internal/vcard" + "github.com/openclaw/crawlkit/control" ) var Version = "dev" @@ -414,10 +419,122 @@ func (c *SearchCmd) Run(r *Runtime) error { type ImportCmd struct { Apple ImportAppleCmd `cmd:"" help:"Import Apple Contacts into local markdown"` Birdclaw ImportBirdclawCmd `cmd:"" help:"Import X/Twitter DM contacts from local birdclaw archive"` + Contacts ImportContactsCmd `cmd:"" help:"Import contacts from a source crawler"` Google ImportGoogleCmd `cmd:"" help:"Import Google Contacts into local markdown"` Discrawl ImportDiscrawlCmd `cmd:"" help:"Import Discord DM contacts from local discrawl archive"` } +type ImportContactsCmd struct { + From string `name:"from" help:"Crawler binary to import contacts from" required:""` +} + +func (c *ImportContactsCmd) Run(r *Runtime) error { + source, contacts, err := readCrawlerContacts(r.ctx, c.From) + if err != nil { + return err + } + changes, err := r.store.ImportCrawlerContacts(source, contacts, r.root.DryRun, time.Now()) + if err != nil { + return err + } + return r.print(changes) +} + +func readCrawlerContacts(ctx context.Context, binary string) (string, []model.SourceContact, error) { + manifest, err := readCrawlerManifest(ctx, binary) + if err != nil { + return "", nil, err + } + command, ok := manifest.Commands["contact-export"] + if !ok { + return "", nil, fmt.Errorf("%s metadata does not advertise contact-export", binary) + } + if !command.JSON { + return "", nil, fmt.Errorf("%s contact-export must advertise json output", binary) + } + if command.Mutates { + return "", nil, fmt.Errorf("%s contact-export must be read-only", binary) + } + if len(command.Argv) == 0 { + return "", nil, fmt.Errorf("%s contact-export argv is empty", binary) + } + argv, err := contactExportArgv(binary, command.Argv) + if err != nil { + return "", nil, err + } + cmd := exec.CommandContext(ctx, argv[0], argv[1:]...) // #nosec G204 -- argv comes from the local crawler manifest and is executed without a shell. + var stderr bytes.Buffer + cmd.Stderr = &stderr + data, err := cmd.Output() + if err != nil { + msg := strings.TrimSpace(stderr.String()) + if msg != "" { + return "", nil, fmt.Errorf("%s contact-export failed: %w: %s", binary, err, msg) + } + return "", nil, fmt.Errorf("%s contact-export failed: %w", binary, err) + } + export, err := contactexport.Decode(bytes.NewReader(data)) + if err != nil { + return "", nil, fmt.Errorf("%s contact-export decode failed: %w", binary, err) + } + source := strings.TrimSpace(manifest.ID) + if source == "" { + source = filepath.Base(binary) + } + return source, sourceContactsFromExport(source, export), nil +} + +func contactExportArgv(binary string, advertised []string) ([]string, error) { + if len(advertised) == 0 { + return nil, fmt.Errorf("%s contact-export argv is empty", binary) + } + requestedName := filepath.Base(binary) + advertisedName := filepath.Base(advertised[0]) + if requestedName != "" && advertisedName != "" && requestedName != advertisedName { + return nil, fmt.Errorf("%s contact-export argv starts with %q, want %q", binary, advertised[0], requestedName) + } + if !slices.Contains(advertised, "--json") { + return nil, fmt.Errorf("%s contact-export argv must include --json", binary) + } + argv := append([]string(nil), advertised...) + argv[0] = binary + return argv, nil +} + +func readCrawlerManifest(ctx context.Context, binary string) (control.Manifest, error) { + cmd := exec.CommandContext(ctx, binary, "--json", "metadata") // #nosec G204 -- binary is an explicit local crawler command. + var stderr bytes.Buffer + cmd.Stderr = &stderr + data, err := cmd.Output() + if err != nil { + msg := strings.TrimSpace(stderr.String()) + if msg != "" { + return control.Manifest{}, fmt.Errorf("%s metadata failed: %w: %s", binary, err, msg) + } + return control.Manifest{}, fmt.Errorf("%s metadata failed: %w", binary, err) + } + var manifest control.Manifest + if err := json.Unmarshal(data, &manifest); err != nil { + return control.Manifest{}, fmt.Errorf("%s metadata decode failed: %w", binary, err) + } + if strings.TrimSpace(manifest.SchemaVersion) != control.SchemaVersion { + return control.Manifest{}, fmt.Errorf("%s metadata schema_version = %q, want %q", binary, manifest.SchemaVersion, control.SchemaVersion) + } + return manifest, nil +} + +func sourceContactsFromExport(source string, export contactexport.ContactExport) []model.SourceContact { + contacts := make([]model.SourceContact, 0, len(export.Contacts)) + for _, c := range export.Contacts { + contact := model.SourceContact{Source: source, Name: c.DisplayName} + for i, phone := range c.PhoneNumbers { + contact.Phones = append(contact.Phones, model.ContactValue{Value: phone, Source: source, Primary: i == 0}) + } + contacts = append(contacts, contact) + } + return contacts +} + type ImportAppleCmd struct { Input string `name:"input" help:"JSON/NDJSON contact file instead of macOS Contacts"` Avatars bool `name:"avatars" help:"Import local avatar thumbnails"` diff --git a/internal/cli/cli_test.go b/internal/cli/cli_test.go index 7c1b82c..d88d662 100644 --- a/internal/cli/cli_test.go +++ b/internal/cli/cli_test.go @@ -15,6 +15,7 @@ import ( "testing" "time" + "github.com/openclaw/clawdex/internal/contactexport" "github.com/openclaw/clawdex/internal/model" ) @@ -285,6 +286,260 @@ func TestExecuteImportDiscrawlErrors(t *testing.T) { } } +func TestExecuteImportContactsFromCrawler(t *testing.T) { + cfg, data := testPaths(t) + var out, errOut bytes.Buffer + if err := Execute([]string{"--config", cfg, "init", data, "--remote", ""}, &out, &errOut); err != nil { + t.Fatal(err) + } + fake := writeFakeContactCrawler(t, "telecrawl", `{"contacts":[{"display_name":"Ada Source","phone_numbers":[" +1 555 0100 "]}]}`) + t.Setenv("PATH", filepath.Dir(fake)+string(os.PathListSeparator)+os.Getenv("PATH")) + out.Reset() + errOut.Reset() + if err := Execute([]string{"--config", cfg, "--dry-run", "import", "contacts", "--from", "telecrawl"}, &out, &errOut); err != nil { + t.Fatalf("import contacts: %v stderr=%s stdout=%s", err, errOut.String(), out.String()) + } + if !strings.Contains(out.String(), "create\tAda Source") { + t.Fatalf("import contacts out = %s", out.String()) + } +} + +func TestExecuteImportContactsFromCrawlerPath(t *testing.T) { + cfg, data := testPaths(t) + var out, errOut bytes.Buffer + if err := Execute([]string{"--config", cfg, "init", data, "--remote", ""}, &out, &errOut); err != nil { + t.Fatal(err) + } + fake := writeFakeContactCrawler(t, "telecrawl", `{"contacts":[{"display_name":"Ada Path","phone_numbers":["123"]}]}`) + out.Reset() + errOut.Reset() + if err := Execute([]string{"--config", cfg, "--dry-run", "import", "contacts", "--from", fake}, &out, &errOut); err != nil { + t.Fatalf("import contacts: %v stderr=%s stdout=%s", err, errOut.String(), out.String()) + } + if !strings.Contains(out.String(), "create\tAda Path") { + t.Fatalf("import contacts out = %s", out.String()) + } +} + +func TestExecuteImportContactsNoopJSONIsEmptyArray(t *testing.T) { + cfg, data := testPaths(t) + var out, errOut bytes.Buffer + if err := Execute([]string{"--config", cfg, "init", data, "--remote", ""}, &out, &errOut); err != nil { + t.Fatal(err) + } + fake := writeFakeContactCrawler(t, "telecrawl", `{"contacts":[{"display_name":"Ada Source","phone_numbers":["+1 555 0100"]}]}`) + out.Reset() + errOut.Reset() + if err := Execute([]string{"--config", cfg, "import", "contacts", "--from", fake}, &out, &errOut); err != nil { + t.Fatalf("first import contacts: %v stderr=%s stdout=%s", err, errOut.String(), out.String()) + } + out.Reset() + errOut.Reset() + if err := Execute([]string{"--config", cfg, "--json", "import", "contacts", "--from", fake}, &out, &errOut); err != nil { + t.Fatalf("second import contacts: %v stderr=%s stdout=%s", err, errOut.String(), out.String()) + } + var changes []model.ImportChange + if err := json.Unmarshal(out.Bytes(), &changes); err != nil { + t.Fatalf("noop import output is not an array: %s", out.String()) + } + if len(changes) != 0 { + t.Fatalf("noop import changes = %#v", changes) + } +} + +func TestExecuteImportContactsDoesNotShellExpandManifestArgv(t *testing.T) { + cfg, data := testPaths(t) + var out, errOut bytes.Buffer + if err := Execute([]string{"--config", cfg, "init", data, "--remote", ""}, &out, &errOut); err != nil { + t.Fatal(err) + } + dir := t.TempDir() + fake := filepath.Join(dir, "telecrawl") + manifest := `{"schema_version":"crawlkit.control.v1","id":"telecrawl","display_name":"Fake Crawler","binary":{"name":"telecrawl"},"commands":{"contact-export":{"argv":["telecrawl","--json","contacts","export;echo shell-expanded"],"json":true}},"privacy":{"contains_private_messages":true,"exports_secrets":false}}` + script := "#!/bin/sh\n" + + "if [ \"$1\" = \"--json\" ] && [ \"$2\" = \"metadata\" ]; then\n" + + "cat <<'JSON'\n" + manifest + "\nJSON\n" + + "exit 0\n" + + "fi\n" + + "if [ \"$1\" = \"--json\" ] && [ \"$2\" = \"contacts\" ] && [ \"$3\" = \"export;echo shell-expanded\" ]; then\n" + + "cat <<'JSON'\n{\"contacts\":[{\"display_name\":\"Ada Argv\",\"phone_numbers\":[\"123\"]}]}\nJSON\n" + + "exit 0\n" + + "fi\n" + + "echo unexpected args: \"$@\" >&2\nexit 2\n" + if err := os.WriteFile(fake, []byte(script), 0o700); err != nil { + t.Fatal(err) + } + out.Reset() + errOut.Reset() + if err := Execute([]string{"--config", cfg, "--dry-run", "import", "contacts", "--from", fake}, &out, &errOut); err != nil { + t.Fatalf("import contacts: %v stderr=%s stdout=%s", err, errOut.String(), out.String()) + } + if !strings.Contains(out.String(), "create\tAda Argv") { + t.Fatalf("import contacts out = %s", out.String()) + } +} + +func TestExecuteImportContactsRejectsMutatingCommand(t *testing.T) { + cfg, data := testPaths(t) + var out, errOut bytes.Buffer + if err := Execute([]string{"--config", cfg, "init", data, "--remote", ""}, &out, &errOut); err != nil { + t.Fatal(err) + } + fake := writeFakeContactCrawlerManifest(t, "telecrawl", `{"schema_version":"crawlkit.control.v1","id":"telecrawl","display_name":"Telegram Crawl","binary":{"name":"telecrawl"},"commands":{"contact-export":{"argv":["telecrawl","--json","contacts","export"],"json":true,"mutates":true}},"privacy":{"contains_private_messages":true,"exports_secrets":false}}`, `{"contacts":[]}`) + t.Setenv("PATH", filepath.Dir(fake)+string(os.PathListSeparator)+os.Getenv("PATH")) + if err := Execute([]string{"--config", cfg, "--dry-run", "import", "contacts", "--from", "telecrawl"}, &out, &errOut); err == nil { + t.Fatal("expected mutating command error") + } +} + +func TestExecuteImportContactsRejectsBadManifests(t *testing.T) { + for _, tc := range []struct { + name string + manifest string + }{ + { + name: "wrong schema", + manifest: `{"schema_version":"not-crawlkit","id":"telecrawl","display_name":"Telegram Crawl","binary":{"name":"telecrawl"},"commands":{"contact-export":{"argv":["telecrawl","--json","contacts","export"],"json":true}},"privacy":{"contains_private_messages":true,"exports_secrets":false}}`, + }, + { + name: "missing command", + manifest: `{"schema_version":"crawlkit.control.v1","id":"telecrawl","display_name":"Telegram Crawl","binary":{"name":"telecrawl"},"commands":{},"privacy":{"contains_private_messages":true,"exports_secrets":false}}`, + }, + { + name: "not json", + manifest: `{"schema_version":"crawlkit.control.v1","id":"telecrawl","display_name":"Telegram Crawl","binary":{"name":"telecrawl"},"commands":{"contact-export":{"argv":["telecrawl","contacts","export"]}},"privacy":{"contains_private_messages":true,"exports_secrets":false}}`, + }, + { + name: "json command missing json flag", + manifest: `{"schema_version":"crawlkit.control.v1","id":"telecrawl","display_name":"Telegram Crawl","binary":{"name":"telecrawl"},"commands":{"contact-export":{"argv":["telecrawl","contacts","export"],"json":true}},"privacy":{"contains_private_messages":true,"exports_secrets":false}}`, + }, + { + name: "empty argv", + manifest: `{"schema_version":"crawlkit.control.v1","id":"telecrawl","display_name":"Telegram Crawl","binary":{"name":"telecrawl"},"commands":{"contact-export":{"argv":[],"json":true}},"privacy":{"contains_private_messages":true,"exports_secrets":false}}`, + }, + } { + t.Run(tc.name, func(t *testing.T) { + cfg, data := testPaths(t) + var out, errOut bytes.Buffer + if err := Execute([]string{"--config", cfg, "init", data, "--remote", ""}, &out, &errOut); err != nil { + t.Fatal(err) + } + fake := writeFakeContactCrawlerManifest(t, "telecrawl", tc.manifest, `{"contacts":[]}`) + t.Setenv("PATH", filepath.Dir(fake)+string(os.PathListSeparator)+os.Getenv("PATH")) + out.Reset() + errOut.Reset() + if err := Execute([]string{"--config", cfg, "--dry-run", "import", "contacts", "--from", "telecrawl"}, &out, &errOut); err == nil { + t.Fatal("expected bad manifest error") + } + }) + } +} + +func TestExecuteImportContactsRejectsBadPayload(t *testing.T) { + cfg, data := testPaths(t) + var out, errOut bytes.Buffer + if err := Execute([]string{"--config", cfg, "init", data, "--remote", ""}, &out, &errOut); err != nil { + t.Fatal(err) + } + fake := writeFakeContactCrawler(t, "telecrawl", `{"contacts":[]} private junk`) + t.Setenv("PATH", filepath.Dir(fake)+string(os.PathListSeparator)+os.Getenv("PATH")) + out.Reset() + errOut.Reset() + if err := Execute([]string{"--config", cfg, "--dry-run", "import", "contacts", "--from", "telecrawl"}, &out, &errOut); err == nil { + t.Fatal("expected bad payload error") + } +} + +func TestExecuteImportContactsRejectsDifferentManifestBinary(t *testing.T) { + cfg, data := testPaths(t) + var out, errOut bytes.Buffer + if err := Execute([]string{"--config", cfg, "init", data, "--remote", ""}, &out, &errOut); err != nil { + t.Fatal(err) + } + fake := writeFakeContactCrawlerManifest(t, "telecrawl", `{"schema_version":"crawlkit.control.v1","id":"telecrawl","display_name":"Telegram Crawl","binary":{"name":"telecrawl"},"commands":{"contact-export":{"argv":["othercrawl","--json","contacts","export"],"json":true}},"privacy":{"contains_private_messages":true,"exports_secrets":false}}`, `{"contacts":[]}`) + out.Reset() + errOut.Reset() + if err := Execute([]string{"--config", cfg, "--dry-run", "import", "contacts", "--from", fake}, &out, &errOut); err == nil { + t.Fatal("expected mismatched manifest binary error") + } +} + +func TestReadCrawlerManifestErrors(t *testing.T) { + dir := t.TempDir() + failing := filepath.Join(dir, "failing") + if err := os.WriteFile(failing, []byte("#!/bin/sh\necho metadata failed >&2\nexit 7\n"), 0o700); err != nil { + t.Fatal(err) + } + if _, err := readCrawlerManifest(t.Context(), failing); err == nil { + t.Fatal("expected metadata command error") + } + + badJSON := filepath.Join(dir, "badjson") + if err := os.WriteFile(badJSON, []byte("#!/bin/sh\necho not-json\n"), 0o700); err != nil { + t.Fatal(err) + } + if _, err := readCrawlerManifest(t.Context(), badJSON); err == nil { + t.Fatal("expected metadata decode error") + } +} + +func TestReadCrawlerContactsReportsExportFailure(t *testing.T) { + dir := t.TempDir() + fake := filepath.Join(dir, "telecrawl") + manifest := `{"schema_version":"crawlkit.control.v1","id":"telecrawl","display_name":"Fake Crawler","binary":{"name":"telecrawl"},"commands":{"contact-export":{"argv":["telecrawl","--json","contacts","export"],"json":true}},"privacy":{"contains_private_messages":true,"exports_secrets":false}}` + script := "#!/bin/sh\n" + + "if [ \"$1\" = \"--json\" ] && [ \"$2\" = \"metadata\" ]; then\n" + + "cat <<'JSON'\n" + manifest + "\nJSON\n" + + "exit 0\n" + + "fi\n" + + "echo export failed >&2\nexit 9\n" + if err := os.WriteFile(fake, []byte(script), 0o700); err != nil { + t.Fatal(err) + } + if _, _, err := readCrawlerContacts(t.Context(), fake); err == nil { + t.Fatal("expected export command error") + } +} + +func TestContactExportArgv(t *testing.T) { + got, err := contactExportArgv("/tmp/telecrawl", []string{"telecrawl", "--json", "contacts", "export"}) + if err != nil { + t.Fatal(err) + } + if got[0] != "/tmp/telecrawl" || got[1] != "--json" { + t.Fatalf("argv = %#v", got) + } + if _, err := contactExportArgv("telecrawl", nil); err == nil { + t.Fatal("expected empty argv error") + } + if _, err := contactExportArgv("telecrawl", []string{"othercrawl"}); err == nil { + t.Fatal("expected mismatched argv error") + } + if _, err := contactExportArgv("telecrawl", []string{"telecrawl", "contacts", "export"}); err == nil { + t.Fatal("expected missing json flag error") + } +} + +func TestSourceContactsFromExportMapsPhones(t *testing.T) { + contacts := sourceContactsFromExport("telecrawl", contactexport.ContactExport{Contacts: []contactexport.Contact{{ + DisplayName: "Ada", + PhoneNumbers: []string{"123", "456"}, + }}}) + if len(contacts) != 1 { + t.Fatalf("contacts = %#v", contacts) + } + got := contacts[0] + if got.Source != "telecrawl" || got.Name != "Ada" || len(got.Phones) != 2 { + t.Fatalf("mapped contact = %#v", got) + } + if !got.Phones[0].Primary || got.Phones[1].Primary { + t.Fatalf("primary phones = %#v", got.Phones) + } + if got.Phones[1].Value != "456" || got.Phones[1].Source != "telecrawl" { + t.Fatalf("second phone = %#v", got.Phones[1]) + } +} + func TestExecuteImportBirdclawErrors(t *testing.T) { cfg, data := testPaths(t) var out, errOut bytes.Buffer @@ -664,6 +919,32 @@ func writeFakeSQLite(t *testing.T, output string) string { return path } +func writeFakeContactCrawler(t *testing.T, name, contacts string) string { + t.Helper() + manifest := `{"schema_version":"crawlkit.control.v1","id":"` + name + `","display_name":"Fake Crawler","binary":{"name":"` + name + `"},"commands":{"contact-export":{"argv":["` + name + `","--json","contacts","export"],"json":true}},"privacy":{"contains_private_messages":true,"exports_secrets":false}}` + return writeFakeContactCrawlerManifest(t, name, manifest, contacts) +} + +func writeFakeContactCrawlerManifest(t *testing.T, name, manifest, contacts string) string { + t.Helper() + dir := t.TempDir() + path := filepath.Join(dir, name) + script := "#!/bin/sh\n" + + "if [ \"$1\" = \"--json\" ] && [ \"$2\" = \"metadata\" ]; then\n" + + "cat <<'JSON'\n" + manifest + "\nJSON\n" + + "exit 0\n" + + "fi\n" + + "if [ \"$1\" = \"--json\" ] && [ \"$2\" = \"contacts\" ] && [ \"$3\" = \"export\" ]; then\n" + + "cat <<'JSON'\n" + contacts + "\nJSON\n" + + "exit 0\n" + + "fi\n" + + "echo unexpected args: \"$@\" >&2\nexit 2\n" + if err := os.WriteFile(path, []byte(script), 0o700); err != nil { + t.Fatal(err) + } + return path +} + func runShell(t *testing.T, dir string, name string, args ...string) { t.Helper() cmd := exec.CommandContext(t.Context(), name, args...) diff --git a/internal/contactexport/contact_export_v0.proto b/internal/contactexport/contact_export_v0.proto new file mode 100644 index 0000000..e30b215 --- /dev/null +++ b/internal/contactexport/contact_export_v0.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; + +package openclaw.clawdex.contactexport.v0; + +message ContactExport { + repeated Contact contacts = 1; +} + +message Contact { + string display_name = 1; + repeated string phone_numbers = 2; +} diff --git a/internal/contactexport/json.go b/internal/contactexport/json.go new file mode 100644 index 0000000..c99bdbf --- /dev/null +++ b/internal/contactexport/json.go @@ -0,0 +1,81 @@ +package contactexport + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "strings" +) + +type ContactExport struct { + Contacts []Contact `json:"contacts"` +} + +type Contact struct { + DisplayName string `json:"display_name"` + PhoneNumbers []string `json:"phone_numbers"` +} + +func Decode(r io.Reader) (ContactExport, error) { + var out ContactExport + dec := json.NewDecoder(r) + dec.DisallowUnknownFields() + if err := dec.Decode(&out); err != nil { + return ContactExport{}, err + } + var extra any + if err := dec.Decode(&extra); err != io.EOF { + if err == nil { + return ContactExport{}, errors.New("contact export must contain exactly one JSON value") + } + return ContactExport{}, err + } + if err := out.Normalize(); err != nil { + return ContactExport{}, err + } + return out, nil +} + +func (e *ContactExport) Normalize() error { + if e == nil { + return errors.New("contact export is nil") + } + if e.Contacts == nil { + return errors.New("contact export missing contacts") + } + contacts := e.Contacts[:0] + for i := range e.Contacts { + c := e.Contacts[i] + name := strings.TrimSpace(c.DisplayName) + phones := cleanPhones(c.PhoneNumbers) + if name == "" { + return fmt.Errorf("contact %d missing display_name", i) + } + if len(phones) == 0 { + return fmt.Errorf("contact %q missing phone_numbers", name) + } + c.DisplayName = name + c.PhoneNumbers = phones + contacts = append(contacts, c) + } + e.Contacts = contacts + return nil +} + +func cleanPhones(values []string) []string { + out := make([]string, 0, len(values)) + seen := map[string]struct{}{} + for _, value := range values { + value = strings.TrimSpace(value) + if value == "" { + continue + } + if _, ok := seen[value]; ok { + continue + } + seen[value] = struct{}{} + out = append(out, value) + } + return out +} diff --git a/internal/contactexport/json_test.go b/internal/contactexport/json_test.go new file mode 100644 index 0000000..e0db57f --- /dev/null +++ b/internal/contactexport/json_test.go @@ -0,0 +1,43 @@ +package contactexport + +import ( + "strings" + "testing" +) + +func TestDecodeNormalizesContacts(t *testing.T) { + got, err := Decode(strings.NewReader(`{"contacts":[{"display_name":" Ada Lovelace ","phone_numbers":[" +1 555 0100 ","","+1 555 0100"]}]}`)) + if err != nil { + t.Fatal(err) + } + if len(got.Contacts) != 1 { + t.Fatalf("contacts = %#v", got.Contacts) + } + if got.Contacts[0].DisplayName != "Ada Lovelace" { + t.Fatalf("name = %q", got.Contacts[0].DisplayName) + } + if got.Contacts[0].PhoneNumbers[0] != "+1 555 0100" || len(got.Contacts[0].PhoneNumbers) != 1 { + t.Fatalf("phones = %#v", got.Contacts[0].PhoneNumbers) + } +} + +func TestDecodeRejectsBadContacts(t *testing.T) { + for _, input := range []string{ + `{`, + `{}`, + `{"contacts":null}`, + `{"contacts":[{}]}`, + `{"contacts":[{"display_name":"Ada","phone_numbers":[]}]}`, + `{"contacts":[{"display_name":"","phone_numbers":["123"]}]}`, + `{"contacts":[{"display_name":"Ada","phone_numbers":["123"],"extra":"x"}]}`, + `{"contacts":[]}{"contacts":[]}`, + `{"contacts":[]} +private junk`, + } { + t.Run(input, func(t *testing.T) { + if _, err := Decode(strings.NewReader(input)); err == nil { + t.Fatal("expected error") + } + }) + } +} diff --git a/internal/index/import.go b/internal/index/import.go index d8acb87..8c60e5b 100644 --- a/internal/index/import.go +++ b/internal/index/import.go @@ -2,6 +2,7 @@ package index import ( "path/filepath" + "reflect" "sort" "strings" "time" @@ -11,27 +12,44 @@ import ( "github.com/openclaw/clawdex/internal/model" ) +type importOptions struct { + DryRun bool + MatchNames bool + TrackSources bool +} + func (s Store) ImportContacts(source string, contacts []model.SourceContact, dryRun bool, now time.Time) ([]model.ImportChange, error) { + return s.importContacts(source, contacts, importOptions{DryRun: dryRun, MatchNames: true}, now) +} + +func (s Store) ImportCrawlerContacts(source string, contacts []model.SourceContact, dryRun bool, now time.Time) ([]model.ImportChange, error) { + return s.importContacts(source, contacts, importOptions{DryRun: dryRun, TrackSources: true}, now) +} + +func (s Store) importContacts(source string, contacts []model.SourceContact, opts importOptions, now time.Time) ([]model.ImportChange, error) { people, err := s.People() if err != nil { return nil, err } - var changes []model.ImportChange + changes := make([]model.ImportChange, 0) for _, contact := range contacts { contact.Source = source if strings.TrimSpace(contact.Name) == "" { continue } - idx := matchContact(people, contact) + idx := matchContact(people, contact, opts.MatchNames) if idx < 0 { p := markdown.NewPerson(contact.Name, now) p.Tags = cleanList(contact.Tags) - p.Emails = sourceValues(contact.Emails, source) - p.Phones = sourceValues(contact.Phones, source) + p.Emails = sourceValues(contact.Emails, source, model.NormalizeEmail) + p.Phones = sourceValues(contact.Phones, source, model.NormalizePhone) p.Accounts = cleanAccounts(contact.Accounts) + if opts.TrackSources { + p.Sources = mergePersonSources(p.Sources, source, contact) + } setExternal(&p, source, contact, now) change := model.ImportChange{Action: "create", PersonID: p.ID, Name: p.Name, Source: contact} - if !dryRun { + if !opts.DryRun { created, err := s.createImportedPerson(p) if err != nil { return nil, err @@ -52,6 +70,8 @@ func (s Store) ImportContacts(source string, contacts []model.SourceContact, dry change.PersonID = created.ID change.Path = created.Path people = append(people, created) + } else { + people = append(people, p) } changes = append(changes, change) continue @@ -61,16 +81,24 @@ func (s Store) ImportContacts(source string, contacts []model.SourceContact, dry beforePhones := len(p.Phones) beforeTags := append([]string(nil), p.Tags...) beforeAccounts := cloneAccounts(p.Accounts) + beforeSources := cloneSources(p.Sources) beforeApple := p.Apple beforeGoogle := p.Google beforeAvatar := p.Avatar + matchedContact := contact + if opts.TrackSources { + matchedContact = contactForPerson(people, idx, contact) + } p.Tags = appendMissingStrings(p.Tags, contact.Tags) - p.Emails = appendMissingValues(p.Emails, contact.Emails, source) - p.Phones = appendMissingValues(p.Phones, contact.Phones, source) + p.Emails = appendMissingValues(p.Emails, matchedContact.Emails, source, model.NormalizeEmail) + p.Phones = appendMissingValues(p.Phones, matchedContact.Phones, source, model.NormalizePhone) p.Accounts = mergeAccounts(p.Accounts, contact.Accounts) + if opts.TrackSources { + p.Sources = mergePersonSources(p.Sources, source, matchedContact) + } setExternal(&p, source, contact, now) avatarChanged := avatarWouldChange(beforeAvatar, contact.Avatar, source) - if !dryRun && contact.Avatar != nil { + if !opts.DryRun && contact.Avatar != nil { var err error p, avatarChanged, err = avatar.SetImported(p, *contact.Avatar, source, now) if err != nil { @@ -80,25 +108,53 @@ func (s Store) ImportContacts(source string, contacts []model.SourceContact, dry externalChanged := p.Apple != beforeApple || p.Google != beforeGoogle tagsChanged := strings.Join(beforeTags, "\x00") != strings.Join(p.Tags, "\x00") accountsChanged := !accountsEqual(beforeAccounts, p.Accounts) - if len(p.Emails) == beforeEmails && len(p.Phones) == beforePhones && !tagsChanged && !accountsChanged && !externalChanged && !avatarChanged { + sourcesChanged := !reflect.DeepEqual(beforeSources, p.Sources) + if len(p.Emails) == beforeEmails && len(p.Phones) == beforePhones && !tagsChanged && !accountsChanged && !sourcesChanged && !externalChanged && !avatarChanged { continue } - change := model.ImportChange{Action: "update", PersonID: p.ID, Name: p.Name, Source: contact, Path: p.Path} - if !dryRun { + change := model.ImportChange{Action: "update", PersonID: p.ID, Name: p.Name, Source: matchedContact, Path: p.Path} + if !opts.DryRun { p.UpdatedAt = now.UTC() if err := markdown.WritePerson(p.Path, p); err != nil { return nil, err } - people[idx] = p } + people[idx] = p changes = append(changes, change) } - if !dryRun { + if !opts.DryRun { return changes, s.Rebuild() } return changes, nil } +func contactForPerson(people []model.Person, idx int, contact model.SourceContact) model.SourceContact { + contact.Emails = contactValuesForPerson(people, idx, contact.Emails, model.NormalizeEmail, personHasEmail) + contact.Phones = contactValuesForPerson(people, idx, contact.Phones, model.NormalizePhone, personHasPhone) + return contact +} + +func contactValuesForPerson(people []model.Person, idx int, values []model.ContactValue, normalize func(string) string, has func(model.Person, string) bool) []model.ContactValue { + out := make([]model.ContactValue, 0, len(values)) + for _, value := range values { + key := normalize(value.Value) + if key == "" || valueOwnedByOtherPerson(people, idx, key, has) { + continue + } + out = append(out, value) + } + return out +} + +func valueOwnedByOtherPerson(people []model.Person, idx int, key string, has func(model.Person, string) bool) bool { + for i, person := range people { + if i != idx && has(person, key) { + return true + } + } + return false +} + func avatarWouldChange(current model.AvatarRef, incoming *model.SourceAvatar, source string) bool { if incoming == nil || len(incoming.Data) == 0 { return false @@ -117,7 +173,7 @@ func avatarWouldChange(current model.AvatarRef, incoming *model.SourceAvatar, so return current.Path == "" || current.Source == "" || current.Source == source } -func matchContact(people []model.Person, contact model.SourceContact) int { +func matchContact(people []model.Person, contact model.SourceContact, matchNames bool) int { for i, p := range people { if accountsOverlap(p.Accounts, contact.Accounts) { return i @@ -149,6 +205,9 @@ func matchContact(people []model.Person, contact model.SourceContact) int { } } } + if !matchNames { + return -1 + } for i, p := range people { if model.NormalizeName(p.Name) != "" && model.NormalizeName(p.Name) == model.NormalizeName(contact.Name) { return i @@ -157,17 +216,23 @@ func matchContact(people []model.Person, contact model.SourceContact) int { return -1 } -func sourceValues(values []model.ContactValue, source string) []model.ContactValue { +func sourceValues(values []model.ContactValue, source string, normalize func(string) string) []model.ContactValue { out := make([]model.ContactValue, 0, len(values)) - for i, value := range values { + seen := map[string]bool{} + for _, value := range values { + key := normalize(value.Value) + if key == "" || seen[key] { + continue + } value.Source = source if value.Label == "" { value.Label = "other" } - if i == 0 { + if len(out) == 0 { value.Primary = true } out = append(out, value) + seen[key] = true } return out } @@ -204,6 +269,21 @@ func cloneAccounts(accounts map[string][]string) map[string][]string { return out } +func cloneSources(sources map[string]model.PersonSource) map[string]model.PersonSource { + if len(sources) == 0 { + return nil + } + out := make(map[string]model.PersonSource, len(sources)) + for source, value := range sources { + out[source] = model.PersonSource{ + Names: append([]string(nil), value.Names...), + Emails: append([]string(nil), value.Emails...), + Phones: append([]string(nil), value.Phones...), + } + } + return out +} + func mergeAccounts(existing map[string][]string, incoming map[string][]string) map[string][]string { if len(incoming) == 0 { return existing @@ -267,22 +347,15 @@ func accountsEqual(a, b map[string][]string) bool { return true } -func appendMissingValues(existing []model.ContactValue, incoming []model.ContactValue, source string) []model.ContactValue { +func appendMissingValues(existing []model.ContactValue, incoming []model.ContactValue, source string, normalize func(string) string) []model.ContactValue { for _, value := range incoming { - key := model.NormalizeEmail(value.Value) - if key == "" { - key = model.NormalizePhone(value.Value) - } + key := normalize(value.Value) if key == "" { continue } found := false for _, cur := range existing { - curKey := model.NormalizeEmail(cur.Value) - if curKey == "" { - curKey = model.NormalizePhone(cur.Value) - } - if curKey == key { + if normalize(cur.Value) == key { found = true break } @@ -298,6 +371,54 @@ func appendMissingValues(existing []model.ContactValue, incoming []model.Contact return existing } +func mergePersonSources(existing map[string]model.PersonSource, source string, contact model.SourceContact) map[string]model.PersonSource { + source = strings.TrimSpace(strings.ToLower(source)) + if source == "" { + return existing + } + if existing == nil { + existing = map[string]model.PersonSource{} + } + current := existing[source] + current.Names = appendMissingNormalizedStrings(current.Names, []string{contact.Name}, model.NormalizeName) + current.Emails = appendMissingContactValues(current.Emails, contact.Emails, model.NormalizeEmail) + current.Phones = appendMissingContactValues(current.Phones, contact.Phones, model.NormalizePhone) + if len(current.Names) == 0 && len(current.Emails) == 0 && len(current.Phones) == 0 { + delete(existing, source) + return existing + } + existing[source] = current + return existing +} + +func appendMissingContactValues(existing []string, incoming []model.ContactValue, normalize func(string) string) []string { + values := make([]string, 0, len(incoming)) + for _, value := range incoming { + values = append(values, value.Value) + } + return appendMissingNormalizedStrings(existing, values, normalize) +} + +func appendMissingNormalizedStrings(existing []string, incoming []string, normalize func(string) string) []string { + seen := map[string]bool{} + for _, value := range existing { + if key := normalize(value); key != "" { + seen[key] = true + } + } + for _, value := range incoming { + value = strings.TrimSpace(value) + key := normalize(value) + if value == "" || key == "" || seen[key] { + continue + } + existing = append(existing, value) + seen[key] = true + } + sort.Strings(existing) + return existing +} + func setExternal(p *model.Person, source string, contact model.SourceContact, now time.Time) { switch source { case "apple": diff --git a/internal/index/store_test.go b/internal/index/store_test.go index d145302..1b2c6e1 100644 --- a/internal/index/store_test.go +++ b/internal/index/store_test.go @@ -214,6 +214,197 @@ func TestImportMatchesEmail(t *testing.T) { } } +func TestCrawlerImportDedupePhonesAndRecordsSources(t *testing.T) { + r := testRepo(t) + s := New(r) + now := time.Now() + if _, err := s.AddPerson("Ada Lovelace", nil, []string{"+31 6 1234 5678"}, nil, now); err != nil { + t.Fatal(err) + } + changes, err := s.ImportCrawlerContacts("telecrawl", []model.SourceContact{{ + Name: "Ada Telegram", + Phones: []model.ContactValue{{Value: "31612345678"}}, + }}, false, now) + if err != nil { + t.Fatal(err) + } + if len(changes) != 1 || changes[0].Action != "update" { + t.Fatalf("changes = %#v", changes) + } + p, err := s.FindPerson("31612345678") + if err != nil { + t.Fatal(err) + } + if len(p.Phones) != 1 { + t.Fatalf("phones = %#v", p.Phones) + } + if got := p.Sources["telecrawl"]; len(got.Names) != 1 || got.Names[0] != "Ada Telegram" || len(got.Phones) != 1 || got.Phones[0] != "31612345678" { + t.Fatalf("telecrawl source = %#v", got) + } + + changes, err = s.ImportCrawlerContacts("wacrawl", []model.SourceContact{{ + Name: "Ada WhatsApp", + Phones: []model.ContactValue{{Value: "+31612345678"}}, + }}, false, now) + if err != nil { + t.Fatal(err) + } + if len(changes) != 1 || changes[0].Action != "update" { + t.Fatalf("changes = %#v", changes) + } + p, err = s.FindPerson("+31612345678") + if err != nil { + t.Fatal(err) + } + if len(p.Phones) != 1 { + t.Fatalf("phones after wacrawl = %#v", p.Phones) + } + if got := p.Sources["wacrawl"]; len(got.Names) != 1 || got.Names[0] != "Ada WhatsApp" || len(got.Phones) != 1 || got.Phones[0] != "+31612345678" { + t.Fatalf("wacrawl source = %#v", got) + } + + changes, err = s.ImportCrawlerContacts("wacrawl", []model.SourceContact{{ + Name: "Ada WhatsApp", + Phones: []model.ContactValue{{Value: "+31612345678"}}, + }}, false, now) + if err != nil { + t.Fatal(err) + } + if len(changes) != 0 { + t.Fatalf("second wacrawl import changes = %#v", changes) + } +} + +func TestCrawlerImportDoesNotMatchByNameOnly(t *testing.T) { + r := testRepo(t) + s := New(r) + now := time.Now() + existing, err := s.AddPerson("Common Name", nil, []string{"+1 555 0100"}, nil, now) + if err != nil { + t.Fatal(err) + } + changes, err := s.ImportCrawlerContacts("telecrawl", []model.SourceContact{{ + Name: "Common Name", + Phones: []model.ContactValue{{Value: "+1 555 0101"}}, + }}, false, now) + if err != nil { + t.Fatal(err) + } + if len(changes) != 1 || changes[0].Action != "create" { + t.Fatalf("changes = %#v", changes) + } + created, err := s.FindPerson("+1 555 0101") + if err != nil { + t.Fatal(err) + } + if created.ID == existing.ID { + t.Fatalf("crawler import name-only merged into existing person: %#v", created) + } +} + +func TestCrawlerImportCreateDedupeNormalizedPhoneValues(t *testing.T) { + r := testRepo(t) + s := New(r) + now := time.Now() + changes, err := s.ImportCrawlerContacts("telecrawl", []model.SourceContact{{ + Name: "Duplicate Phone", + Phones: []model.ContactValue{ + {Value: "+1 555 0100"}, + {Value: "15550100"}, + }, + }}, false, now) + if err != nil { + t.Fatal(err) + } + if len(changes) != 1 || changes[0].Action != "create" { + t.Fatalf("changes = %#v", changes) + } + p, err := s.FindPerson("+1 555 0100") + if err != nil { + t.Fatal(err) + } + if len(p.Phones) != 1 { + t.Fatalf("phones = %#v", p.Phones) + } + if got := p.Sources["telecrawl"]; len(got.Phones) != 1 || got.Phones[0] != "+1 555 0100" { + t.Fatalf("telecrawl source = %#v", got) + } +} + +func TestCrawlerImportSkipsPhoneOwnedByAnotherPerson(t *testing.T) { + r := testRepo(t) + s := New(r) + now := time.Now() + ada, err := s.AddPerson("Ada Existing", nil, []string{"+1 555 0100"}, nil, now) + if err != nil { + t.Fatal(err) + } + bob, err := s.AddPerson("Bob Existing", nil, []string{"+1 555 0101"}, nil, now) + if err != nil { + t.Fatal(err) + } + changes, err := s.ImportCrawlerContacts("telecrawl", []model.SourceContact{{ + Name: "Ada Telegram", + Phones: []model.ContactValue{ + {Value: "+1 555 0100"}, + {Value: "+1 555 0101"}, + }, + }}, false, now) + if err != nil { + t.Fatal(err) + } + if len(changes) != 1 || changes[0].Action != "update" { + t.Fatalf("changes = %#v", changes) + } + if len(changes[0].Source.Phones) != 1 || changes[0].Source.Phones[0].Value != "+1 555 0100" { + t.Fatalf("change source phones = %#v", changes[0].Source.Phones) + } + gotAda, err := s.FindPerson("+1 555 0100") + if err != nil { + t.Fatal(err) + } + if gotAda.ID != ada.ID || len(gotAda.Phones) != 1 { + t.Fatalf("ada = %#v", gotAda) + } + if got := gotAda.Sources["telecrawl"]; len(got.Phones) != 1 || got.Phones[0] != "+1 555 0100" { + t.Fatalf("telecrawl source = %#v", got) + } + gotBob, err := s.FindPerson("+1 555 0101") + if err != nil { + t.Fatal(err) + } + if gotBob.ID != bob.ID || len(gotBob.Phones) != 1 { + t.Fatalf("bob = %#v", gotBob) + } +} + +func TestCrawlerImportDryRunMatchesRealDuplicateCollapse(t *testing.T) { + r := testRepo(t) + s := New(r) + now := time.Now() + contacts := []model.SourceContact{ + {Name: "Duplicate Contact", Phones: []model.ContactValue{{Value: "+1 555 0100"}}}, + {Name: "Duplicate Contact", Phones: []model.ContactValue{{Value: "15550100"}}}, + } + changes, err := s.ImportCrawlerContacts("telecrawl", contacts, true, now) + if err != nil { + t.Fatal(err) + } + if len(changes) != 1 || changes[0].Action != "create" { + t.Fatalf("dry-run changes = %#v", changes) + } + if _, err := s.FindPerson("+1 555 0100"); err == nil { + t.Fatal("dry-run created person") + } + changes, err = s.ImportCrawlerContacts("telecrawl", contacts, false, now) + if err != nil { + t.Fatal(err) + } + if len(changes) != 1 || changes[0].Action != "create" { + t.Fatalf("real changes = %#v", changes) + } +} + func TestImportWritesExternalOnlyChange(t *testing.T) { r := testRepo(t) s := New(r) diff --git a/internal/markdown/markdown.go b/internal/markdown/markdown.go index a2ee945..d1f630f 100644 --- a/internal/markdown/markdown.go +++ b/internal/markdown/markdown.go @@ -212,18 +212,19 @@ func inferNote(n *model.Note, path string) { } type personFront struct { - ID string `yaml:"id"` - Name string `yaml:"name"` - SortName string `yaml:"sort_name,omitempty"` - Tags []string `yaml:"tags,omitempty"` - Emails []model.ContactValue `yaml:"emails,omitempty"` - Phones []model.ContactValue `yaml:"phones,omitempty"` - Avatar *model.AvatarRef `yaml:"avatar,omitempty"` - Accounts map[string][]string `yaml:"accounts,omitempty"` - Apple *model.ExternalRef `yaml:"apple,omitempty"` - Google *model.ExternalRef `yaml:"google,omitempty"` - CreatedAt time.Time `yaml:"created_at"` - UpdatedAt time.Time `yaml:"updated_at"` + ID string `yaml:"id"` + Name string `yaml:"name"` + SortName string `yaml:"sort_name,omitempty"` + Tags []string `yaml:"tags,omitempty"` + Emails []model.ContactValue `yaml:"emails,omitempty"` + Phones []model.ContactValue `yaml:"phones,omitempty"` + Avatar *model.AvatarRef `yaml:"avatar,omitempty"` + Accounts map[string][]string `yaml:"accounts,omitempty"` + Sources map[string]model.PersonSource `yaml:"sources,omitempty"` + Apple *model.ExternalRef `yaml:"apple,omitempty"` + Google *model.ExternalRef `yaml:"google,omitempty"` + CreatedAt time.Time `yaml:"created_at"` + UpdatedAt time.Time `yaml:"updated_at"` } func personFrontmatter(p model.Person) personFront { @@ -236,6 +237,7 @@ func personFrontmatter(p model.Person) personFront { Phones: p.Phones, Avatar: nonEmptyAvatar(p.Avatar), Accounts: nonEmptyAccounts(p.Accounts), + Sources: nonEmptySources(p.Sources), Apple: nonEmptyExternal(p.Apple), Google: nonEmptyExternal(p.Google), CreatedAt: p.CreatedAt, @@ -298,6 +300,13 @@ func nonEmptyAccounts(accounts map[string][]string) map[string][]string { return accounts } +func nonEmptySources(sources map[string]model.PersonSource) map[string]model.PersonSource { + if len(sources) == 0 { + return nil + } + return sources +} + func nonZeroTime(t time.Time) *time.Time { if t.IsZero() { return nil diff --git a/internal/markdown/markdown_test.go b/internal/markdown/markdown_test.go index 6da04b2..1745762 100644 --- a/internal/markdown/markdown_test.go +++ b/internal/markdown/markdown_test.go @@ -16,6 +16,7 @@ func TestPersonRoundTrip(t *testing.T) { now := time.Date(2026, 5, 8, 9, 15, 0, 0, time.UTC) p := NewPerson("Ada Lovelace", now) p.Tags = []string{"math"} + p.Sources = map[string]model.PersonSource{"telecrawl": {Names: []string{"Ada Lovelace"}, Phones: []string{"15550100"}}} p.Body = "# Ada Lovelace\n\nNotes." if err := WritePerson(path, p); err != nil { t.Fatal(err) @@ -30,6 +31,9 @@ func TestPersonRoundTrip(t *testing.T) { if got.ID != p.ID || got.Name != p.Name || strings.TrimSpace(got.Body) != strings.TrimSpace(p.Body) { t.Fatalf("roundtrip mismatch: %#v", got) } + if got.Sources["telecrawl"].Phones[0] != "15550100" { + t.Fatalf("sources = %#v", got.Sources) + } } func TestPersonRepairSalvagesBrokenFrontmatter(t *testing.T) { @@ -152,6 +156,7 @@ func TestWritePersonOmitsEmptyStructsButKeepsNonEmpty(t *testing.T) { path := filepath.Join(dir, "person.md") p := NewPerson("Ada", time.Now()) p.Accounts = map[string][]string{"github": {"ada"}} + p.Sources = map[string]model.PersonSource{"telecrawl": {Names: []string{"Ada"}, Phones: []string{"123"}}} p.Avatar.Path = "avatars/avatar.png" p.Avatar.Source = "manual" p.Google.Resource = "people/c1" @@ -163,7 +168,7 @@ func TestWritePersonOmitsEmptyStructsButKeepsNonEmpty(t *testing.T) { t.Fatal(err) } text := string(data) - if !strings.Contains(text, "accounts:") || !strings.Contains(text, "avatar:") || !strings.Contains(text, "google:") || strings.Contains(text, "apple:") { + if !strings.Contains(text, "accounts:") || !strings.Contains(text, "sources:") || !strings.Contains(text, "avatar:") || !strings.Contains(text, "google:") || strings.Contains(text, "apple:") { t.Fatalf("frontmatter = %s", text) } } diff --git a/internal/model/types.go b/internal/model/types.go index de00756..4f089bc 100644 --- a/internal/model/types.go +++ b/internal/model/types.go @@ -26,6 +26,12 @@ type AvatarRef struct { UpdatedAt time.Time `json:"updated_at,omitzero" yaml:"updated_at,omitempty"` } +type PersonSource struct { + Names []string `json:"names,omitempty" yaml:"names,omitempty"` + Emails []string `json:"emails,omitempty" yaml:"emails,omitempty"` + Phones []string `json:"phones,omitempty" yaml:"phones,omitempty"` +} + type Person struct { ID string `json:"id" yaml:"id"` Name string `json:"name" yaml:"name"` @@ -35,6 +41,7 @@ type Person struct { Phones []ContactValue `json:"phones,omitempty" yaml:"phones,omitempty"` Avatar AvatarRef `json:"avatar,omitzero" yaml:"avatar,omitempty"` Accounts map[string][]string `json:"accounts,omitempty" yaml:"accounts,omitempty"` + Sources map[string]PersonSource `json:"sources,omitempty" yaml:"sources,omitempty"` Apple ExternalRef `json:"apple,omitzero" yaml:"apple,omitempty"` Google ExternalRef `json:"google,omitzero" yaml:"google,omitempty"` CreatedAt time.Time `json:"created_at" yaml:"created_at"`