Initial commit

Signed-off-by: Gustavo Sampaio <[email protected]>

Author: GustavoKatel

.gitignore

@@ -0,0 +1 @@
+tmp

.vscode/launch.json

@@ -0,0 +1,18 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Launch",
+            "type": "go",
+            "request": "launch",
+            "mode": "auto",
+            "program": "${fileDirname}",
+            "env": {},
+            "args": [],
+            "cwd": "${workspaceFolder}"
+        }
+    ]
+}

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "go.formatTool": "goimports"
+}

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 Runlet
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,7 @@
+# 🦀 CrabFS
+
+CrabFS is a distributed (p2p) filesystem using the billy filesystem interface.
+
+See [examples](https://github.com/runletapp/crabfs/tree/master/examples) for a concrete example.
+
+## This is a big WIP. Use with caution and PR's are welcome

Taskfile

@@ -0,0 +1,15 @@
+#!/bin/bash
+PATH=./node_modules/.bin:$PATH
+
+function generateProtos {
+    protoc -I ./protobuf ./protobuf/*.proto --go_out=plugins=grpc:./protos
+}
+
+function help {
+    echo "$0 <task> <args>"
+    echo "Tasks:"
+    compgen -A function | cat -n
+}
+
+TIMEFORMAT="Task completed in %3lR"
+time ${@:-default}

block_slicer.go

@@ -0,0 +1,53 @@
+package crabfs
+
+import (
+	"io"
+
+	blocks "github.com/ipfs/go-block-format"
+	"github.com/runletapp/crabfs/interfaces"
+	pb "github.com/runletapp/crabfs/protos"
+)
+
+var _ interfaces.Slicer = &BlockSlicer{}
+
+// BlockSlicer a simple block slicer
+type BlockSlicer struct {
+	reader io.Reader
+
+	blockSize int64
+
+	offset int64
+}
+
+// BlockSlicerNew creates a new BlockSlicer with 'blockSize'
+func BlockSlicerNew(reader io.Reader, blockSize int64) (interfaces.Slicer, error) {
+	slicer := &BlockSlicer{
+		reader:    reader,
+		blockSize: blockSize,
+
+		offset: 0,
+	}
+
+	return slicer, nil
+}
+
+func (slicer *BlockSlicer) Next() (*pb.BlockMetadata, blocks.Block, error) {
+	data := make([]byte, slicer.blockSize)
+
+	n, err := slicer.reader.Read(data)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	block := blocks.NewBlock(data[:n])
+
+	metadata := &pb.BlockMetadata{
+		Start: slicer.offset,
+		Size:  int64(n),
+		Cid:   block.Cid().Bytes(),
+	}
+
+	slicer.offset += int64(n)
+
+	return metadata, block, nil
+}

crabfs.go

@@ -0,0 +1,163 @@
+package crabfs
+
+import (
+	"context"
+	"crypto/rand"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/ipfs/go-cid"
+
+	"github.com/runletapp/crabfs/interfaces"
+	"github.com/runletapp/crabfs/options"
+
+	ipfsDatastore "github.com/ipfs/go-datastore"
+	ipfsDatastoreSync "github.com/ipfs/go-datastore/sync"
+	ipfsBlockstore "github.com/ipfs/go-ipfs-blockstore"
+	libp2pCrypto "github.com/libp2p/go-libp2p-crypto"
+)
+
+var _ interfaces.Core = &crabFS{}
+
+type crabFS struct {
+	settings *options.Settings
+
+	host interfaces.Host
+
+	datastore  *ipfsDatastoreSync.MutexDatastore
+	blockstore ipfsBlockstore.Blockstore
+}
+
+// New create a new CrabFS
+func New(opts ...options.Option) (interfaces.Core, error) {
+	settings := options.Settings{}
+	if err := settings.SetDefaults(); err != nil {
+		return nil, err
+	}
+
+	for _, opt := range opts {
+		if err := opt(&settings); err != nil {
+			return nil, err
+		}
+	}
+
+	if settings.BlockSize == 0 {
+		return nil, ErrInvalidBlockSize
+	}
+
+	var privateKey *libp2pCrypto.RsaPrivateKey
+	var err error
+	if settings.PrivateKey == nil {
+		privKey, _, err := libp2pCrypto.GenerateRSAKeyPair(2048, rand.Reader)
+		if err != nil {
+			return nil, err
+		}
+		privateKey = privKey.(*libp2pCrypto.RsaPrivateKey)
+	} else {
+		privateKey, err = ReadPrivateKey(settings.PrivateKey)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// TODO: Persistence
+	datastore := ipfsDatastoreSync.MutexWrap(ipfsDatastore.NewMapDatastore())
+
+	blockstore := ipfsBlockstore.NewBlockstore(datastore)
+
+	host, err := HostNew(&settings, privateKey, datastore, blockstore)
+	if err != nil {
+		return nil, err
+	}
+
+	if !settings.RelayOnly {
+		if err := host.Announce(); err != nil {
+			return nil, err
+		}
+	}
+
+	fs := &crabFS{
+		settings: &settings,
+
+		host: host,
+
+		datastore: datastore,
+
+		blockstore: blockstore,
+	}
+
+	return fs, nil
+}
+
+func (fs *crabFS) Get(ctx context.Context, filename string) (io.ReadSeeker, int64, error) {
+	blockMap, err := fs.host.GetContent(ctx, filename)
+	if err != nil {
+		return nil, 0, err
+	}
+
+	fetcher, err := BasicFetcherNew(ctx, fs, blockMap)
+	if err != nil {
+		return nil, 0, err
+	}
+
+	return fetcher, fetcher.Size(), nil
+}
+
+func (fs *crabFS) Put(ctx context.Context, filename string, file io.Reader, mtime time.Time) error {
+	slicer, err := BlockSlicerNew(file, fs.settings.BlockSize)
+	if err != nil {
+		return err
+	}
+
+	blockMap := interfaces.BlockMap{}
+	totalSize := int64(0)
+
+	blockMeta, block, err := slicer.Next()
+	for {
+		if err != nil && err != io.EOF {
+			return err
+		}
+
+		if block == nil {
+			break
+		}
+
+		if err := fs.blockstore.Put(block); err != nil {
+			return err
+		}
+
+		blockMap[blockMeta.Start] = blockMeta
+		totalSize += blockMeta.Size
+
+		// Process block
+		blockMeta, block, err = slicer.Next()
+	}
+
+	return fs.host.Publish(ctx, filename, blockMap, mtime, totalSize)
+}
+
+func (fs *crabFS) Remove(ctx context.Context, filename string) error {
+	blockMap, err := fs.host.GetContent(ctx, filename)
+	if err != nil {
+		return err
+	}
+
+	for _, blockMeta := range blockMap {
+		cid, _ := cid.Cast(blockMeta.Cid)
+
+		if err := fs.blockstore.DeleteBlock(cid); err != nil {
+			fmt.Printf("Could not delete block: %s", cid.String())
+		}
+	}
+
+	return nil
+}
+
+func (fs *crabFS) GetID() string {
+	return fs.host.GetID()
+}
+
+func (fs *crabFS) GetAddrs() []string {
+	return fs.host.GetAddrs()
+}

dht_namespace_pk_validator.go

@@ -0,0 +1,57 @@
+package crabfs
+
+import (
+	"fmt"
+	"strings"
+
+	libp2pCrypto "github.com/libp2p/go-libp2p-crypto"
+	libp2pRecord "github.com/libp2p/go-libp2p-record"
+	multihash "github.com/multiformats/go-multihash"
+)
+
+// DHTNamespacePKValidatorNew creates a new validator that validates for all versions
+func DHTNamespacePKValidatorNew() libp2pRecord.Validator {
+	return DHTNamespacePKValidatorV1{}
+}
+
+// DHTNamespacePKValidatorV1 validates the /crabfs keys on the dht datastore
+type DHTNamespacePKValidatorV1 struct {
+}
+
+// Validate validates the given record, returning an error if it's
+// invalid (e.g., expired, signed by the wrong key, etc.).
+func (validator DHTNamespacePKValidatorV1) Validate(key string, value []byte) error {
+	parts := strings.Split(key, "/")
+
+	if len(parts) != 3 {
+		return fmt.Errorf("Invalid key. Expexted format: /crabfs_pk/<hash> Got: %v", key)
+	}
+
+	if _, err := libp2pCrypto.UnmarshalRsaPublicKey(value); err != nil {
+		return err
+	}
+
+	pskHash, err := multihash.Sum(value, multihash.SHA3_256, -1)
+	if err != nil {
+		return err
+	}
+
+	expectedKey := fmt.Sprintf("/crabfs_pk/%s", pskHash.String())
+	if !strings.HasPrefix(key, expectedKey) {
+		return fmt.Errorf("Invalid key. Expexted: %s Got: %v", expectedKey, key)
+	}
+
+	return nil
+}
+
+// Select selects the best record from the set of records (e.g., the
+// newest).
+//
+// Decisions made by select should be stable.
+func (validator DHTNamespacePKValidatorV1) Select(key string, values [][]byte) (int, error) {
+	if len(values) == 0 {
+		return 0, fmt.Errorf("No values")
+	}
+
+	return 0, nil
+}