(WIP) feat(synapse-core): add sp-fetch module for SP-to-SP piece transfers

AGENTS.md

@@ -46,7 +46,7 @@ packages/synapse-sdk/src/
 
 **Commands**:
 
-- Root: `pnpm run fix` (Biome auto-fix all), `pnpm run build` (all packages), `pnpm test`
+- Root: `pnpm run lint:fix` (Biome auto-fix all), `pnpm run build` (all packages), `pnpm test`
 - Package: `pnpm run lint:fix`, `pnpm run build`, `pnpm test` (from `packages/synapse-sdk/`)
 
 **Build**: TypeScript → `dist/` (in package), ES modules with `.js` extensions, strict mode, NodeNext resolution

examples/cli/package.json

@@ -16,8 +16,10 @@
     "@clack/prompts": "^0.11.0",
     "@filoz/synapse-core": "workspace:^",
     "@filoz/synapse-sdk": "workspace:^",
+    "@remix-run/fs": "^0.3.0",
     "cleye": "^2.0.0",
     "conf": "^15.0.2",
+    "terminal-link": "^5.0.0",
     "viem": "catalog:"
   },
   "devDependencies": {

examples/cli/src/client.ts

@@ -22,6 +22,7 @@ export function privateKeyClient(chainId: number) {
     client,
     privateKey: privateKey as Hex,
     rpcURL: chain.rpcUrls.default.http[0],
+    chain,
   }
 }
 

examples/cli/src/commands/datasets-create.ts

@@ -0,0 +1,103 @@
+import * as p from '@clack/prompts'
+import * as sp from '@filoz/synapse-core/sp'
+import {
+  createDataSet,
+  getProvider,
+  readProviders,
+} from '@filoz/synapse-core/warm-storage'
+import { type Command, command } from 'cleye'
+import type { Account, Chain, Client, Transport } from 'viem'
+import { privateKeyClient } from '../client.ts'
+import { globalFlags } from '../flags.ts'
+import { hashLink } from '../utils.ts'
+
+export const datasetsCreate: Command = command(
+  {
+    name: 'datasets-create',
+    description: 'Create a data set',
+    alias: 'dc',
+    parameters: ['[providerId]'],
+    flags: {
+      ...globalFlags,
+      cdn: {
+        type: Boolean,
+        description: 'Enable CDN',
+        default: false,
+      },
+    },
+    help: {
+      description: 'Create a data set',
+    },
+  },
+  async (argv) => {
+    const { client, chain } = privateKeyClient(argv.flags.chain)
+
+    const spinner = p.spinner()
+    try {
+      const provider = argv._.providerId
+        ? await getProvider(client, { providerId: BigInt(argv._.providerId) })
+        : await selectProvider(client, argv.flags)
+
+      p.log.info(
+        `Selected provider: #${provider.id} - ${provider.serviceProvider} ${provider.pdp.serviceURL}`
+      )
+      spinner.start(`Creating data set...`)
+
+      const result = await createDataSet(client, {
+        payee: provider.payee,
+        payer: client.account.address,
+        endpoint: provider.pdp.serviceURL,
+        cdn: argv.flags.cdn,
+      })
+
+      spinner.message(
+        `Waiting for tx ${hashLink(result.txHash, chain)} to be mined...`
+      )
+      const dataset = await sp.waitForDataSetCreationStatus(result)
+
+      spinner.stop(`Data set created #${dataset.dataSetId}`)
+    } catch (error) {
+      spinner.stop('Failed to create data set', 1)
+      if (argv.flags.debug) {
+        console.error(error)
+      } else {
+        p.log.error((error as Error).message)
+      }
+      process.exit(1)
+    }
+  }
+)
+
+async function selectProvider(
+  client: Client<Transport, Chain, Account>,
+  options: { debug?: boolean }
+) {
+  const spinner = p.spinner()
+  spinner.start(`Fetching providers...`)
+
+  try {
+    const providers = await readProviders(client)
+    spinner.stop(`Fetching providers complete`)
+
+    const provider = await p.select({
+      message: 'Pick a provider to create a data set.',
+      options: providers.map((provider) => ({
+        value: provider,
+        label: `#${provider.id} - ${provider.serviceProvider} ${provider.pdp.serviceURL}`,
+      })),
+    })
+    if (p.isCancel(provider)) {
+      p.cancel('Operation cancelled.')
+      process.exit(1)
+    }
+    return provider
+  } catch (error) {
+    spinner.stop('Failed to select data set', 1)
+    if (options.debug) {
+      console.error(error)
+    } else {
+      p.log.error((error as Error).message)
+    }
+    process.exit(1)
+  }
+}

examples/cli/src/commands/datasets-terminate.ts

@@ -0,0 +1,90 @@
+import * as p from '@clack/prompts'
+import { getDataSets, terminateDataSet } from '@filoz/synapse-core/warm-storage'
+import { type Command, command } from 'cleye'
+import type { Account, Chain, Client, Transport } from 'viem'
+import { waitForTransactionReceipt } from 'viem/actions'
+import { privateKeyClient } from '../client.ts'
+import { globalFlags } from '../flags.ts'
+import { hashLink } from '../utils.ts'
+
+export const datasetsTerminate: Command = command(
+  {
+    name: 'datasets-terminate',
+    description: 'Terminate a data set',
+    alias: 'dt',
+    parameters: ['[dataSetId]'],
+    flags: {
+      ...globalFlags,
+    },
+    help: {
+      description: 'Terminate a data set',
+    },
+  },
+  async (argv) => {
+    const { client, chain } = privateKeyClient(argv.flags.chain)
+
+    const spinner = p.spinner()
+    try {
+      const dataSetId = argv._.dataSetId
+        ? BigInt(argv._.dataSetId)
+        : await selectDataSet(client, argv.flags)
+      spinner.start(`Terminating data set ${dataSetId}...`)
+
+      const tx = await terminateDataSet(client, {
+        dataSetId: BigInt(dataSetId),
+      })
+
+      spinner.message(`Waiting for tx ${hashLink(tx, chain)} to be mined...`)
+      await waitForTransactionReceipt(client, {
+        hash: tx,
+      })
+
+      spinner.stop(`Data set terminated`)
+    } catch (error) {
+      spinner.stop('Failed to terminate data set', 1)
+      if (argv.flags.debug) {
+        console.error(error)
+      } else {
+        p.log.error((error as Error).message)
+      }
+      process.exit(1)
+    }
+  }
+)
+
+async function selectDataSet(
+  client: Client<Transport, Chain, Account>,
+  options: { debug?: boolean }
+) {
+  const spinner = p.spinner()
+  spinner.start(`Fetching data sets...`)
+
+  try {
+    const dataSets = await getDataSets(client, {
+      address: client.account.address,
+    })
+    spinner.stop(`Fetching data sets complete`)
+
+    const dataSetId = await p.select({
+      message: 'Pick a data set to terminate.',
+      options: dataSets.map((dataSet) => ({
+        value: dataSet.dataSetId,
+        label: `#${dataSet.dataSetId} - SP: #${dataSet.providerId} ${dataSet.pdp.serviceURL} ${dataSet.pdpEndEpoch > 0n ? `Terminating at epoch ${dataSet.pdpEndEpoch}` : ''}`,
+      })),
+    })
+    if (p.isCancel(dataSetId)) {
+      p.cancel('Operation cancelled.')
+      process.exit(1)
+    }
+
+    return dataSetId
+  } catch (error) {
+    spinner.stop('Failed to select data set', 1)
+    if (options.debug) {
+      console.error(error)
+    } else {
+      p.log.error((error as Error).message)
+    }
+    process.exit(1)
+  }
+}

examples/cli/src/commands/datasets.ts

@@ -1,6 +1,7 @@
 import * as p from '@clack/prompts'
 import { getDataSets } from '@filoz/synapse-core/warm-storage'
 import { type Command, command } from 'cleye'
+import { getBlockNumber } from 'viem/actions'
 import { privateKeyClient } from '../client.ts'
 import { globalFlags } from '../flags.ts'
 
@@ -22,6 +23,8 @@ export const datasets: Command = command(
 
     const spinner = p.spinner()
 
+    const blockNumber = await getBlockNumber(client)
+
     spinner.start('Listing data sets...')
     try {
       const dataSets = await getDataSets(client, {
@@ -30,10 +33,11 @@ export const datasets: Command = command(
       spinner.stop('Data sets:')
       dataSets.forEach(async (dataSet) => {
         p.log.info(
-          `#${dataSet.dataSetId} ${dataSet.cdn ? 'CDN' : ''} ${dataSet.pdp.serviceURL} ${dataSet.pdpEndEpoch > 0n ? `Terminating at epoch ${dataSet.pdpEndEpoch}` : ''}`
+          `#${dataSet.dataSetId} ${dataSet.cdn ? 'CDN' : ''} ${dataSet.pdp.serviceURL} ${dataSet.pdpEndEpoch > 0n ? `Terminating at epoch ${dataSet.pdpEndEpoch}` : ''} ${dataSet.live ? 'Live' : ''} ${dataSet.managed ? 'Managed' : ''}`
         )
-        console.log(dataSet)
+        // console.log(dataSet)
       })
+      p.log.warn(`Block number: ${blockNumber}`)
     } catch (error) {
       spinner.stop()
       console.error(error)

examples/cli/src/commands/pieces-upload.ts

@@ -0,0 +1,58 @@
+import path from 'node:path'
+import * as p from '@clack/prompts'
+import * as SP from '@filoz/synapse-core/sp'
+import { upload } from '@filoz/synapse-core/warm-storage'
+import { openFile } from '@remix-run/fs'
+import { type Command, command } from 'cleye'
+import { privateKeyClient } from '../client.ts'
+import { globalFlags } from '../flags.ts'
+import { hashLink } from '../utils.ts'
+
+export const piecesUpload: Command = command(
+  {
+    name: 'pieces-upload',
+    parameters: ['<path>', '<dataSetId>'],
+    description: 'Upload a file to a data set',
+    flags: {
+      ...globalFlags,
+      cdn: {
+        type: Boolean,
+        description: 'Enable CDN',
+        default: false,
+      },
+    },
+    help: {
+      description: 'Upload a file to a data set',
+    },
+  },
+  async (argv) => {
+    const { client, chain } = privateKeyClient(argv.flags.chain)
+    const spinner = p.spinner()
+
+    const filePath = argv._.path
+    const absolutePath = path.resolve(filePath)
+    const file = openFile(absolutePath)
+
+    spinner.start(`Uploading file ${absolutePath}...`)
+    try {
+      const result = await upload(client, {
+        dataSetId: BigInt(argv._.dataSetId),
+        data: [file],
+        onEvent: (event, data) => {
+          spinner.message(`${event} ${data.pieceCid.toString()}`)
+        },
+      })
+
+      spinner.message(
+        `Waiting for tx ${hashLink(result.txHash, chain)} to be mined...`
+      )
+      const pieces = await SP.waitForAddPiecesStatus(result)
+      spinner.stop(`File uploaded ${pieces.confirmedPieceIds.join(',')}`)
+    } catch (error) {
+      spinner.stop()
+      p.log.error((error as Error).message)
+      p.outro('Please try again')
+      return
+    }
+  }
+)

examples/cli/src/commands/upload-dataset.ts

@@ -72,7 +72,7 @@ export const uploadDataset: Command = command(
         ],
       })
 
-      await SP.pollForDataSetCreationStatus(rsp)
+      await SP.waitForDataSetCreationStatus(rsp)
       spinner.stop(`File uploaded ${pieceCid}`)
     } catch (error) {
       spinner.stop()