This commit is contained in:
Ehmry - 2018-09-07 15:34:25 +02:00
parent a163f062d3
commit 02c6fddad6
33 changed files with 2202 additions and 56 deletions

12
dagfs.nimble Normal file
View File

@ -0,0 +1,12 @@
# Package
version = "0.1.1"
author = "Emery Hemingway"
description = "A simple content addressed file-system"
license = "GPLv3"
srcDir = "src"
requires "nim >= 0.18.0", "base58", "cbor >= 0.2.0"
bin = @["dagfs_repl.nim"]
skipFiles = @["dagfs_repl.nim"]

BIN
dagfs_repl.nim Executable file

Binary file not shown.

View File

@ -0,0 +1,136 @@
when not defined(genode):
{.error: "Genode only Dagfs client".}
import cbor, genode, std/tables, std/strutils
import dagfs, dagfs/stores, dagfs/genode/dagfs_session
const
currentPath = currentSourcePath.rsplit("/", 1)[0]
dagfsClientH = currentPath & "/dagfs_client.h"
{.passC: "-I" & currentPath & "/../../../genode/include".}
type
DagfsClientBase {.importcpp, header: dagfsClientH.} = object
DagfsClientCpp = Constructible[DagfsClientBase]
proc sigh_ack_avail(cpp: DagfsClientCpp; sig: SignalContextCapability) {.
importcpp: "#->conn.channel().sigh_ack_avail(@)", tags: [RpcEffect].}
proc readyToSubmit(cpp: DagfsClientCpp): bool {.
importcpp: "#->conn.source().ready_to_submit()".}
proc readyToAck(cpp: DagfsClientCpp): bool {.
importcpp: "#->conn.source().ready_to_ack()".}
proc ackAvail(cpp: DagfsClientCpp): bool {.
importcpp: "#->conn.source().ack_avail()".}
proc allocPacket(cpp: DagfsClientCpp; size = MaxPacketSize): DagfsPacket {.
importcpp: "#->conn.source().alloc_packet(@)".}
proc packetContent(cpp: DagfsClientCpp; pkt: DagfsPacket): pointer {.
importcpp: "#->conn.source().packet_content(@)".}
proc submitPacket(cpp: DagfsClientCpp; pkt: DagfsPacket; cid: cstring; op: DagfsOpcode) {.
importcpp: "#->conn.source().submit_packet(Dagfs::Packet(#, (char const *)#, #))".}
proc getAckedPacket(cpp: DagfsClientCpp): DagfsPacket {.
importcpp: "#->conn.source().get_acked_packet()".}
proc releasePacket(cpp: DagfsClientCpp; pkt: DagfsPacket) {.
importcpp: "#->conn.source().release_packet(@)".}
type
DagfsClient* = ref DagfsClientObj
DagfsClientObj = object of DagfsStoreObj
## IPLD session client
cpp: DagfsClientCpp
proc icClose(s: DagfsStore) =
var ic = DagfsClient(s)
destruct ic.cpp
proc icPut(s: DagfsStore; blk: string): Cid =
## Put block to Dagfs server, blocks for two packet round-trip.
let ic = DagfsClient(s)
var
blk = blk
pktCid = dagHash blk
if pktCid == zeroBlock:
return pktCid
assert(ic.cpp.readyToSubmit, "Dagfs client packet queue congested")
var pkt = ic.cpp.allocPacket(blk.len)
let pktBuf = ic.cpp.packetContent pkt
defer: ic.cpp.releasePacket pkt
assert(not pktBuf.isNil, "allocated packet has nil content")
assert(pkt.size >= blk.len)
pkt.setLen blk.len
copyMem(pktBuf, blk[0].addr, blk.len)
assert(ic.cpp.readyToSubmit, "Dagfs client packet queue congested")
ic.cpp.submitPacket(pkt, pktCid.toHex, PUT)
let ack = ic.cpp.getAckedPacket()
doAssert(ack.error == OK)
result = ack.cid()
assert(result.isValid, "server returned a packet with and invalid CID")
proc icGetBuffer(s: DagfsStore; cid: Cid; buf: pointer; len: Natural): int =
## Get from Dagfs server, blocks for packet round-trip.
let ic = DagfsClient(s)
assert(ic.cpp.readyToSubmit, "Dagfs client packet queue congested")
let pkt = ic.cpp.allocPacket len
ic.cpp.submitPacket(pkt, cid.toHex, GET)
let ack = ic.cpp.getAckedPacket
doAssert(ack.cid == cid)
if ack.error == OK:
let pktBuf = ic.cpp.packetContent ack
assert(not pktBuf.isNil, "ack packet has nil content")
assert(ack.len <= len)
assert(ack.len > 0)
result = ack.len
copyMem(buf, pktBuf, result)
if pkt.size > 0:
ic.cpp.releasePacket pkt
# free the original packet that was allocated
case ack.error:
of OK: discard
of MISSING:
raise cid.newMissingObject
else:
raise newException(CatchableError, "Dagfs packet error " & $ack.error)
proc icGet(s: DagfsStore; cid: Cid; result: var string) =
## Get from Dagfs server, blocks for packet round-trip.
let ic = DagfsClient(s)
assert(ic.cpp.readyToSubmit, "Dagfs client packet queue congested")
let pkt = ic.cpp.allocPacket()
defer: ic.cpp.releasePacket pkt
ic.cpp.submitPacket(pkt, cid.toHex, GET)
let ack = ic.cpp.getAckedPacket()
doAssert(ack.cid == cid)
case ack.error:
of OK:
let ackBuf = ic.cpp.packetContent ack
assert(not ackBuf.isNil)
assert(ack.len > 0)
result.setLen ack.len
copyMem(result[0].addr, ackBuf, result.len)
assert(cid.verify(result), "Dagfs client packet failed verification")
of MISSING:
raise cid.newMissingObject
else:
raise newException(CatchableError, "Dagfs packet error " & $ack.error)
const
DefaultDagfsBufferSize* = 1 shl 20
proc newDagfsClient*(env: GenodeEnv; label = ""; bufferSize = DefaultDagfsBufferSize): DagfsClient =
## Blocks retrieved by `get` are not verified.
proc construct(cpp: DagfsClientCpp; env: GenodeEnv; label: cstring; txBufSize: int) {.
importcpp.}
new result
construct(result.cpp, env, label, bufferSize)
result.closeImpl = icClose
result.putImpl = icPut
result.getBufferImpl = icGetBuffer
result.getImpl = icGet

View File

@ -0,0 +1,56 @@
/*
* \brief IPLD C++ session component
* \author Emery Hemingway
* \date 2017-11-07
*/
/*
* Copyright (C) 2017 Genode Labs GmbH
*
* This file is part of the Genode OS framework, which is distributed
* under the terms of the GNU Affero General Public License version 3.
*/
#ifndef _INCLUDE__NIM__IPLDSERVER_H_
#define _INCLUDE__NIM__IPLDSERVER_H_
#include <ipld_session/rpc_object.h>
#include <base/heap.h>
#include <base/attached_ram_dataspace.h>
struct Communication_buffer
{
Genode::Attached_ram_dataspace _tx_ds;
Communication_buffer(Genode::Pd_session &pd,
Genode::Region_map &rm,
Genode::size_t tx_buf_size)
: _tx_ds(pd, rm, tx_buf_size) { }
};
struct IpldSessionComponentBase : Communication_buffer,
Ipld::Session_rpc_object
{
static Genode::size_t tx_buf_size(char const *args)
{
Genode::size_t const buf_size = Genode::Arg_string::find_arg(
args, "tx_buf_size").aligned_size();
if (!buf_size)
throw Genode::Service_denied();
return buf_size;
}
IpldSessionComponentBase(Genode::Env *env, char const *args)
:
Communication_buffer(env->pd(), env->rm(), tx_buf_size(args)),
Session_rpc_object(env->rm(), env->ep().rpc_ep(), _tx_ds.cap())
{ }
void packetHandler(Genode::Signal_context_capability cap)
{
_tx.sigh_ready_to_ack(cap);
_tx.sigh_packet_avail(cap);
}
};
#endif /* _INCLUDE__NIM__IPLDSERVER_H_ */

View File

@ -0,0 +1,142 @@
#
# \brief IPLD server factory
# \author Emery Hemingway
# \date 2017-11-11
#
#
# Copyright (C) 2017 Genode Labs GmbH
#
# This file is part of the Genode OS framework, which is distributed
# under the terms of the GNU Affero General Public License version 3.
#
import std/strtabs, std/tables, std/xmltree, std/strutils
import cbor, genode, genode/signals, genode/servers, ipld, ipld/store, ipldsession
const
currentPath = currentSourcePath.rsplit("/", 1)[0]
ipldserverH = currentPath & "/ipldserver.h"
type
IpldSessionComponentBase {.importcpp, header: ipldserverH.} = object
SessionCpp = Constructible[IpldSessionComponentBase]
Session = ref object
cpp: SessionCpp
sig: SignalHandler
store: IpldStore
id: SessionId
label: string
proc processPacket(session: Session; pkt: var IpldPacket) =
proc packetContent(cpp: SessionCpp; pkt: IpldPacket): pointer {.
importcpp: "#->sink().packet_content(@)".}
let cid = pkt.cid
case pkt.operation
of PUT:
try:
var
pktBuf = session.cpp.packetContent pkt
heapBuf = newString pkt.len
copyMem(heapBuf[0].addr, pktBuf, heapBuf.len)
let putCid = session.store.put(heapBuf, cid.hash)
assert(putCid.isValid, "server packet returned invalid CID from put")
pkt.setCid putCid
except:
echo "unhandled PUT error ", getCurrentExceptionMsg()
pkt.setError ERROR
of GET:
try:
let
pktBuf = session.cpp.packetContent pkt
n = session.store.getBuffer(cid, pktBuf, pkt.size)
pkt.setLen n
except BufferTooSmall:
pkt.setError OVERSIZE
except MissingObject:
pkt.setError MISSING
except:
echo "unhandled GET error ", getCurrentExceptionMsg()
pkt.setError ERROR
else:
echo "invalid packet operation"
pkt.setError ERROR
proc newSession(env: GenodeEnv; store: IpldStore; id: SessionId; label, args: string): Session =
## Create a new session and packet handling procedure
let session = new Session
assert(not session.isNil)
proc construct(cpp: SessionCpp; env: GenodeEnv; args: cstring) {.importcpp.}
session.cpp.construct(env, args)
session.store = store
session.id = id
session.label = label
session.sig = env.ep.newSignalHandler do ():
proc packetAvail(cpp: SessionCpp): bool {.
importcpp: "#->sink().packet_avail()".}
proc readyToAck(cpp: SessionCpp): bool {.
importcpp: "#->sink().ready_to_ack()".}
while session.cpp.packetAvail and session.cpp.readyToAck:
proc getPacket(cpp: SessionCpp): IpldPacket {.
importcpp: "#->sink().get_packet()".}
var pkt = session.cpp.getPacket()
session.processPacket pkt
proc acknowledgePacket(cpp: SessionCpp; pkt: IpldPacket) {.
importcpp: "#->sink().acknowledge_packet(@)".}
session.cpp.acknowledgePacket(pkt)
proc packetHandler(cpp: SessionCpp; cap: SignalContextCapability) {.
importcpp: "#->packetHandler(@)".}
session.cpp.packetHandler(session.sig.cap)
result = session
proc manage(ep: Entrypoint; s: Session): IpldSessionCapability =
## Manage a session from the default entrypoint.
proc manage(ep: Entrypoint; cpp: SessionCpp): IpldSessionCapability {.
importcpp: "#.manage(*#)".}
result = ep.manage(s.cpp)
GC_ref s
proc dissolve(ep: Entrypoint; s: Session) =
## Dissolve a session from the entrypoint so that it can be freed.
proc dissolve(ep: Entrypoint; cpp: SessionCpp) {.
importcpp: "#.dissolve(*#)".}
ep.dissolve(s.cpp)
destruct(s.cpp)
dissolve(s.sig)
GC_unref s
type
IpldServer* = ref object
env: GenodeEnv
store*: IpldStore
sessions*: Table[SessionId, Session]
proc newIpldServer*(env: GenodeEnv; store: IpldStore): IpldServer =
IpldServer(
env: env, store: store,
sessions: initTable[SessionId, Session]())
proc create*(server: IpldServer; id: SessionId; label, args: string) =
if not server.sessions.contains id:
try:
let
session = newSession(server.env, server.store, id, label, args)
cap = server.env.ep.manage(session)
server.sessions[id] = session
proc deliverSession(env: GenodeEnv; id: SessionId; cap: IpldSessionCapability) {.
importcpp: "#->parent().deliver_session_cap(Genode::Parent::Server::Id{#}, #)".}
server.env.deliverSession(id, cap)
echo "session opened for ", label
except:
echo "failed to create session for '", label, "', ", getCurrentExceptionMsg()
server.env.sessionResponseDeny id
proc close*(server: IpldServer; id: SessionId) =
## Close a session at the IPLD server.
if server.sessions.contains id:
let session = server.sessions[id]
server.env.ep.dissolve(session)
server.sessions.del id
server.env.sessionResponseClose id

View File

@ -0,0 +1,47 @@
#
# \brief IPLD session definitions
# \author Emery Hemingway
# \date 2017-11-11
#
#
# Copyright (C) 2017 Genode Labs GmbH
#
# This file is part of the Genode OS framework, which is distributed
# under the terms of the GNU Affero General Public License version 3.
#
import ipld
const MaxPacketSize* = 1 shl 18;
type
IpldSessionCapability* {.final, pure,
importcpp: "Ipld::Session_capability",
header: "<ipld_session/capability.h>".} = object
IpldPacket* {.
importcpp: "Ipld::Packet",
header: "<ipld_session/ipld_session.h>".} = object
IpldOpcode* {.importcpp: "Ipld::Packet::Opcode".} = enum
PUT, GET, INVALID
IpldError* {.importcpp: "Ipld::Packet::Error".} = enum
OK, MISSING, OVERSIZE, FULL, ERROR
proc size*(pkt: IpldPacket): csize {.importcpp.}
## Physical packet size.
proc cidStr(p: IpldPacket): cstring {.importcpp: "#.cid().string()".}
proc cid*(p: IpldPacket): Cid = parseCid $p.cidStr
proc setCid*(p: var IpldPacket; cid: cstring) {.importcpp: "#.cid(@)".}
proc setCid*(p: var IpldPacket; cid: Cid) = p.setCid(cid.toHex())
proc operation*(pkt: IpldPacket): IpldOpcode {.importcpp.}
proc len*(pkt: IpldPacket): csize {.importcpp: "length".}
## Logical packet length.
proc setLen*(pkt: var IpldPacket; len: int) {.importcpp: "length".}
## Set logical packet length.
proc error*(pkt: IpldPacket): IpldError {.importcpp.}
proc setError*(pkt: var IpldPacket; err: IpldError) {.importcpp: "error".}

View File

@ -0,0 +1,28 @@
/*
* \brief C++ base of Dagfs client
* \author Emery Hemingway
* \date 2017-11-08
*/
/*
* Copyright (C) 2017 Genode Labs GmbH
*
* This file is part of the Genode OS framework, which is distributed
* under the terms of the GNU Affero General Public License version 3.
*/
/* Genode includes */
#include <dagfs_session/connection.h>
#include <base/heap.h>
struct DagfsClientBase
{
Genode::Heap heap;
Genode::Allocator_avl tx_packet_alloc { &heap };
Dagfs::Connection conn;
DagfsClientBase(Genode::Env *env, char const *label, Genode::size_t tx_buf_size)
: heap(env->pd(), env->rm()),
conn(*env, tx_packet_alloc, label, tx_buf_size)
{ }
};

View File

@ -1,56 +0,0 @@
#
# \brief Server-side IPLD session interface
# \author Emery Hemingway
# \date 2017-11-04
#
#
# Copyright (C) 2017 Genode Labs GmbH
#
# This file is part of the Genode OS framework, which is distributed
# under the terms of the GNU Affero General Public License version 3.
#
import xmltree, strtabs, xmlparser, streams, tables,
genode, genode/servers, genode/roms, ipld/genode/ipldserver, ipld/ipfsdaemon
proc newDaemonStore(env: GenodeEnv): IpfsStore =
## Open a connection to an IPFS daemon.
try:
let
configRom = env.newRomClient("config")
config = configRom.xml
close configRom
let daemonUrl = config.attrs["ipfs_url"]
result = newIpfsStore(daemonUrl)
except:
let err = getCurrentException()
quit("failed to connect IPFS, " & err.msg)
componentConstructHook = proc (env: GenodeEnv) =
let
store = env.newDaemonStore()
# Server backend
server = env.newIpldServer(store)
# Store server
proc processSessions(rom: RomClient) =
## ROM signal handling procedure
## Create and close 'Ipld' sessions from the
## 'sessions_requests' ROM.
update rom
var requests = initSessionRequestsParser(rom)
for id in requests.close:
server.close id
for id, service, label in requests.create:
if service == "Ipld":
server.create id, label, requests.args
let sessionsHandle = env.newRomHandler(
"session_requests", processSessions)
env.announce "Ipld"
process sessionsHandle
# Process request backlog and return to the entrypoint.

140
src/dagfs.nim Normal file
View File

@ -0,0 +1,140 @@
import std/hashes, std/streams, std/strutils
import base58/bitcoin, cbor
import ./dagfs/priv/hex, ./dagfs/priv/blake2
const
maxBlockSize* = 1 shl 18
## Maximum supported block size.
digestLen* = 32
## Length of a block digest.
type Cid* = object
## Content IDentifier, used to identify blocks.
digest*: array[digestLen, uint8]
proc initCid*(): Cid = Cid()
## Initialize an invalid CID.
proc isValid*(x: Cid): bool =
## Check that a CID has been properly initialized.
for c in x.digest.items:
if c != 0: return true
proc `==`*(x, y: Cid): bool =
## Compare two CIDs.
for i in 0..<digestLen:
if x.digest[i] != y.digest[i]:
return false
true
proc `==`*(cbor: CborNode; cid: Cid): bool =
## Compare a CBOR node with a CID.
if cbor.kind == cborBytes:
for i in 0..<digestLen:
if cid.digest[i] != cbor.bytes[i].uint8:
return false
result = true
proc hash*(cid: Cid): Hash = hash cid.digest
## Reduce a CID into an integer for use in tables.
proc toCbor*(cid: Cid): CborNode = newCborBytes cid.digest
## Generate a CBOR representation of a CID.
proc toCid*(cbor: CborNode): Cid =
## Generate a CBOR representation of a CID.
assert(cbor.bytes.len == digestLen)
for i in 0..<digestLen:
result.digest[i] = cbor.bytes[i].uint8
{.deprecated: [newCborBytes: toCbor].}
proc toHex*(cid: Cid): string = hex.encode(cid.digest)
## Return CID encoded in hexidecimal.
proc writeUvarint*(s: Stream; n: SomeInteger) =
## Write an IPFS varint
var n = n
while true:
let c = int8(n and 0x7f)
n = n shr 7
if n == 0:
s.write((char)c.char)
break
else:
s.write((char)c or 0x80)
proc readUvarint*(s: Stream): BiggestInt =
## Read an IPFS varint
var shift: int
while shift < (9*8):
let c = (BiggestInt)s.readChar
result = result or ((c and 0x7f) shl shift)
if (c and 0x80) == 0:
break
shift.inc 7
proc toIpfs*(cid: Cid): string =
## Return CID encoded in IPFS multimulti.
const
multiRaw = 0x55
multiBlake2b_256 = 0xb220
let s = newStringStream()
s.writeUvarint 1
s.writeUvarint multiRaw
s.writeUvarint multi_blake2b_256
s.writeUvarint digestLen
for e in cid.digest:
s.write e
s.setPosition 0
result = 'z' & bitcoin.encode(s.readAll)
close s
proc `$`*(cid: Cid): string = toHex cid
## Return CID in base 58, the default textual encoding.
proc parseCid*(s: string): Cid =
## Detect CID encoding and parse from a string.
var raw = parseHexStr s
if raw.len != digestLen:
raise newException(ValueError, "invalid ID length")
for i in 0..<digestLen:
result.digest[i] = raw[i].byte
const
zeroBlock* = parseCid "8ddb61928ec76e4ee904cd79ed977ab6f5d9187f1102975060a6ba6ce10e5481"
## CID of zero block of maximum size.
proc take*(cid: var Cid; buf: var string) =
## Take a raw digest from a string buffer.
doAssert(buf.len == digestLen)
copyMem(cid.digest[0].addr, buf[0].addr, digestLen)
proc dagHash*(data: string): Cid =
## Generate a CID for a string of data using the BLAKE2b hash algorithm.
assert(data.len <= maxBlockSize)
var b: Blake2b
blake2b_init(b, digestLen, nil, 0)
blake2b_update(b, data, data.len)
var s = blake2b_final(b)
copyMem(result.digest[0].addr, s[0].addr, digestLen)
proc verify*(cid: Cid; data: string): bool =
## Verify that a string of data corresponds to a CID.
var b: Blake2b
blake2b_init(b, digestLen, nil, 0)
blake2b_update(b, data, data.len)
let digest = blake2b_final(b)
for i in 0..<digestLen:
if cid.digest[i] != digest[i]:
return false
true
iterator simpleChunks*(s: Stream; size = maxBlockSize): string =
## Iterator that breaks a stream into simple chunks.
doAssert(size <= maxBlockSize)
var tmp = newString(size)
while not s.atEnd:
tmp.setLen(size)
tmp.setLen(s.readData(tmp[0].addr, size))
yield tmp

360
src/dagfs/fsnodes.nim Normal file
View File

@ -0,0 +1,360 @@
import strutils, streams, tables, cbor, os, math
import ../dagfs, ./stores
type EntryKey = enum
typeKey = 1,
dataKey = 2,
sizeKey = 3
type FsType* = enum
ufsFile = 0,
ufsDir = 1
type FsKind* = enum
fileNode,
dirNode,
shallowDir,
shallowFile
type
FileLink* = object
cid*: Cid
size*: int
FsNode* = ref object
cid: Cid
case kind*: FsKind
of fileNode:
links*: seq[FileLink]
of dirNode:
entries: OrderedTable[string, FsNode]
of shallowFile, shallowDir:
discard
size: BiggestInt
proc isRaw*(file: FsNode): bool =
file.links.len == 0
proc cid*(u: FsNode): Cid =
assert u.cid.isValid
u.cid
proc isFile*(u: FsNode): bool = u.kind in { fileNode, shallowFile }
proc isDir*(u: FsNode): bool = u.kind in { dirNode, shallowDir }
proc size*(u: FsNode): BiggestInt =
if u.kind == dirNode: u.entries.len.BiggestInt
else: u.size
proc newFsRoot*(): FsNode =
FsNode(
cid: initCid(),
kind: dirNode,
entries: initOrderedTable[string, FsNode](8))
proc newUnixfsFile*(): FsNode =
FsNode(kind: fileNode, cid: initCid())
proc newUnixfsDir*(cid: Cid): FsNode =
FsNode(cid: cid, kind: dirNode)
proc add*(root: var FsNode; name: string; node: FsNode) =
root.entries[name] = node
proc addDir*(root: var FsNode; name: string; cid: Cid) {.deprecated.} =
assert cid.isValid
root.add name, FsNode(kind: dirNode, cid: cid)
proc addFile*(root: var FsNode; name: string; cid: Cid; size: BiggestInt) {.deprecated.} =
assert cid.isValid
root.add name, FsNode(kind: fileNode, cid: cid, size: size)
proc del*(dir: var FsNode; name: string) =
dir.entries.del name
const
DirTag* = 0xda3c80 ## CBOR tag for UnixFS directories
FileTag* = 0xda3c81 ## CBOR tag for UnixFS files
proc isUnixfs*(bin: string): bool =
## Check if a string contains a UnixFS node
## in CBOR form.
var
s = newStringStream bin
c: CborParser
try:
c.open s
c.next
if c.kind == CborEventKind.cborTag:
result = c.tag == DirTag or c.tag == FileTag
except ValueError: discard
close s
proc toCbor*(u: FsNode): CborNode =
case u.kind
of fileNode:
let array = newCborArray()
array.seq.setLen u.links.len
for i in 0..u.links.high:
let L = newCborMap()
# typeEntry is reserved but not in use
L[dataKey.int] = u.links[i].cid.newCborBytes
L[sizeKey.int] = u.links[i].size.newCborInt
array.seq[i] = L
result = newCborTag(FileTag, array)
of dirNode:
let map = newCborMap()
for name, node in u.entries:
var entry = newCborMap()
case node.kind
of fileNode, shallowFile:
entry[typeKey.int] = ufsFile.int.newCborInt
entry[dataKey.int] = node.cid.newCborBytes
entry[sizeKey.int] = node.size.newCborInt
of dirNode:
entry[typeKey.int] = ufsDir.int.newCborInt
entry[dataKey.int] = node.cid.newCborBytes
entry[sizeKey.int] = node.entries.len.newCborInt
of shallowdir:
entry[typeKey.int] = ufsDir.int.newCborInt
entry[dataKey.int] = node.cid.newCborBytes
entry[sizeKey.int] = node.size.int.newCborInt
map[name] = entry
# TODO: the CBOR maps must be sorted
result = newCborTag(DirTag, map)
else:
raiseAssert "shallow FsNodes can not be encoded"
template parseAssert(cond: bool; msg = "") =
if not cond: raise newException(
ValueError,
if msg == "": "invalid UnixFS CBOR" else: "invalid UnixFS CBOR, " & msg)
proc parseFs*(raw: string; cid: Cid): FsNode =
## Parse a string containing CBOR data into a FsNode.
new result
result.cid = cid
var
c: CborParser
buf = ""
open(c, newStringStream(raw))
next c
parseAssert(c.kind == CborEventKind.cborTag, "data not tagged")
let tag = c.tag
if tag == FileTag:
result.kind = fileNode
next c
parseAssert(c.kind == CborEventKind.cborArray, "file data not an array")
let nLinks = c.arrayLen
result.links = newSeq[FileLink](nLinks)
for i in 0..<nLinks:
next c
parseAssert(c.kind == CborEventKind.cborMap, "file array does not contain maps")
let nAttrs = c.mapLen
for _ in 1..nAttrs:
next c
parseAssert(c.kind == CborEventKind.cborPositive, "link map key not an integer")
let key = c.readInt.EntryKey
next c
case key
of typeKey:
parseAssert(false, "type file links are not supported")
of dataKey:
parseAssert(c.kind == CborEventKind.cborBytes, "CID not encoded as bytes")
c.readBytes buf
result.links[i].cid.take buf
of sizeKey:
parseAssert(c.kind == CborEventKind.cborPositive, "link size not encoded properly")
result.links[i].size = c.readInt
result.size.inc result.links[i].size
elif tag == DirTag:
result.kind = dirNode
next c
parseAssert(c.kind == CborEventKind.cborMap)
let dirLen = c.mapLen
parseAssert(dirLen != -1, raw)
result.entries = initOrderedTable[string, FsNode](dirLen.nextPowerOfTwo)
for i in 1 .. dirLen:
next c
parseAssert(c.kind == CborEventKind.cborText, raw)
c.readText buf
parseAssert(not buf.contains({ '/', '\0'}), raw)
next c
parseAssert(c.kind == CborEventKind.cborMap)
let nAttrs = c.mapLen
parseAssert(nAttrs > 1, raw)
let entry = new FsNode
result.entries[buf] = entry
for i in 1 .. nAttrs:
next c
parseAssert(c.kind == CborEventKind.cborPositive)
case c.readInt.EntryKey
of typeKey:
next c
case c.readInt.FsType
of ufsFile: entry.kind = shallowFile
of ufsDir: entry.kind = shallowDir
of dataKey:
next c
c.readBytes buf
entry.cid.take buf
of sizeKey:
next c
entry.size = c.readInt
else:
parseAssert(false, raw)
next c
parseAssert(c.kind == cborEof, "trailing data")
proc toStream*(node: FsNode; s: Stream) =
let c = node.toCbor()
c.toStream s
iterator items*(dir: FsNode): (string, FsNode) =
assert(dir.kind == dirNode)
for k, v in dir.entries.pairs:
yield (k, v)
proc containsFile*(dir: FsNode; name: string): bool =
doAssert(dir.kind == dirNode)
dir.entries.contains name
proc `[]`*(dir: FsNode; name: string): FsNode =
if dir.kind == dirNode:
result = dir.entries.getOrDefault name
proc `[]`*(dir: FsNode; index: int): (string, FsNode) =
result[0] = ""
if dir.kind == dirNode:
var i = 0
for name, node in dir.entries.pairs:
if i == index:
result = (name, node)
break
inc i
proc lookupFile*(dir: FsNode; name: string): tuple[cid: Cid, size: BiggestInt] =
doAssert(dir.kind == dirNode)
let f = dir.entries[name]
if f.kind == fileNode:
result.cid = f.cid
result.size = f.size
proc addFile*(store: DagfsStore; path: string): FsNode =
## Add a file to the store and a FsNode.
let
fStream = newFileStream(path, fmRead)
u = newUnixfsFile()
u.links = newSeqOfCap[FileLink](1)
for chunk in fStream.simpleChunks:
let cid = store.put(chunk)
u.links.add FileLink(cid: cid, size: chunk.len)
u.size.inc chunk.len
if u.size == 0:
# return the CID for a raw nothing
u.cid = dagHash("")
else:
if u.links.len == 1:
# take a shortcut use the raw chunk CID
u.cid = u.links[0].cid
else:
u.cid = store.putDag(u.toCbor)
result = u
close fStream
proc addDir*(store: DagfsStore; dirPath: string): FsNode =
var dRoot = newFsRoot()
for kind, path in walkDir dirPath:
var child: FsNode
case kind
of pcFile:
child = store.addFile path
of pcDir:
child = store.addDir(path)
else: continue
dRoot.add path.extractFilename, child
let
dag = dRoot.toCbor
cid = store.putDag(dag)
result = newUnixfsDir(cid)
proc open*(store: DagfsStore; cid: Cid): FsNode =
assert cid.isValid
let raw = store.get(cid)
result = parseFs(raw, cid)
proc openDir*(store: DagfsStore; cid: Cid): FsNode =
assert cid.isValid
var raw = ""
try: store.get(cid, raw)
except MissingObject: raise cid.newMissingObject
# this sucks
result = parseFs(raw, cid)
assert(result.kind == dirNode)
proc walk*(store: DagfsStore; dir: FsNode; path: string; cache = true): FsNode =
## Walk a path down a root.
assert(dir.kind == dirNode)
result = dir
var raw = ""
for name in split(path, DirSep):
if name == "": continue
if result.kind == fileNode:
result = nil
break
var next = result[name]
if next.isNil:
result = nil
break
if (next.kind in {shallowFile, shallowDir}):
store.get(next.cid, raw)
next = parseFs(raw, next.cid)
if cache:
result.entries[name] = next
result = next
#[
iterator fileChunks*(store: DagfsStore; file: FsNode): string =
## Iterate over the links in a file and return futures for link data.
if file.cid.isRaw:
yield store.get(file.cid)
else:
var
i = 0
chunk = ""
while i < file.links.len:
store.get(file.links[i].cid, chunk)
yield chunk
inc i
]#
proc readBuffer*(store: DagfsStore; file: FsNode; pos: BiggestInt;
buf: pointer; size: int): int =
## Read a UnixFS file into a buffer. May return zero for any failure.
assert(pos > -1)
var
filePos = 0
chunk = ""
if pos < file.size:
#[
if file.cid.isRaw:
let pos = pos.int
store.get(file.cid, chunk)
if pos < chunk.high:
copyMem(buf, chunk[pos].addr, min(chunk.len - pos, size))
result = size
else:
]#
block:
for i in 0..file.links.high:
let linkSize = file.links[i].size
if filePos <= pos and pos < filePos+linkSize:
store.get(file.links[i].cid, chunk)
let
chunkPos = int(pos - filePos)
n = min(chunk.len-chunkPos, size)
copyMem(buf, chunk[chunkPos].addr, n)
result = n
break
filePos.inc linkSize

80
src/dagfs/ipfsdaemon.nim Normal file
View File

@ -0,0 +1,80 @@
import httpclient, json, base58/bitcoin, streams, cbor, tables
import ../dagfs, ./stores, ./fsnodes
type
IpfsStore* = ref IpfsStoreObj
IpfsStoreObj = object of DagfsStoreObj
## IPFS daemon client.
http: HttpClient
baseUrl: string
proc ipfsClose(s: DagfsStore) =
var ipfs = IpfsStore(s)
close ipfs.http
proc putBlock(ipfs: IpfsStore; data: string; format = "raw"): tuple[key: string, size: int] =
# stuff in some MIME horseshit so it works
ipfs.http.headers = newHttpHeaders({
"Content-Type": "multipart/form-data; boundary=------------------------KILL_A_WEBDEV"})
let
trash = """
--------------------------KILL_A_WEBDEV
Content-Disposition: form-data; name="file"; filename="myfile"
Content-Type: application/octet-stream
""" & data & """
--------------------------KILL_A_WEBDEV--
"""
resp = ipfs.http.post(ipfs.baseUrl & "/api/v0/block/put?format=" & format, body=trash)
body = resp.body
js = parseJson body
# You can tell its written in Go when the JSON keys had to be capitalized
result = (js["Key"].getStr, js["Size"].getInt)
proc ipfsPut(s: DagfsStore; blk: string): Cid =
var ipfs = IpfsStore(s)
let
isDag = blk.isUnixfs
tag = 0x55
format = "raw"
result = CidBlake2b256 blk
discard ipfs.putBlock(blk, format)
# IPFS returns a different hash. Whatever.
proc ipfsGetBuffer(s: DagfsStore; cid: Cid; buf: pointer; len: Natural): int =
var ipfs = IpfsStore(s)
let url = ipfs.baseUrl & "/api/v0/block/get?arg=" & $cid
try:
var body = ipfs.http.request(url).body
if not verify(cid, body):
raise newMissingObject cid
if body.len > len:
raise newException(BufferTooSmall, "")
result = body.len
copyMem(buf, body[0].addr, result)
except:
raise newMissingObject cid
proc ipfsGet(s: DagfsStore; cid: Cid; result: var string) =
var ipfs = IpfsStore(s)
let url = ipfs.baseUrl & "/api/v0/block/get?arg=" & $cid
try:
result = ipfs.http.request(url).body
if not verify(cid, result):
raise newMissingObject cid
except:
raise newMissingObject cid
proc newIpfsStore*(url = "http://127.0.0.1:5001"): IpfsStore =
## Allocate a new synchronous store interface to the IPFS daemon at `url`.
## Every block retrieved by `get` is hashed and verified.
new result
result.closeImpl = ipfsClose
result.putImpl = ipfsPut
result.getBufferImpl = ipfsGetBuffer
result.getImpl = ipfsGet
result.http = newHttpClient()
result.baseUrl = url

165
src/dagfs/priv/blake2.nim Normal file
View File

@ -0,0 +1,165 @@
type
Blake2b* = object
hash: array[8, uint64]
offset: array[2, uint64]
buffer: array[128, uint8]
buffer_idx: uint8
hash_size: uint8
const Blake2bIV =
[ 0x6a09e667f3bcc908'u64, 0xbb67ae8584caa73b'u64,
0x3c6ef372fe94f82b'u64, 0xa54ff53a5f1d36f1'u64,
0x510e527fade682d1'u64, 0x9b05688c2b3e6c1f'u64,
0x1f83d9abfb41bd6b'u64, 0x5be0cd19137e2179'u64 ]
const Sigma = [
[ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 ],
[ 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 ],
[ 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 ],
[ 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 ],
[ 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 ],
[ 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 ],
[ 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 ],
[ 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 ],
[ 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 ],
[ 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0 ],
[ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 ],
[ 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 ] ]
proc inc(a: var array[2, uint64], b: uint8) =
a[0] = a[0] + b
if (a[0] < b): inc(a[1])
proc padding(a: var array[128, uint8], b: uint8) =
for i in b..127: a[i] = 0
proc ror64(x: uint64, n: int): uint64 {.inline.} =
result = (x shr n) or (x shl (64 - n))
proc G (v: var array[16, uint64],
a,b,c,d: int, x,y: uint64)
{.inline.} =
v[a] = v[a] + v[b] + x
v[d] = ror64(v[d] xor v[a], 32)
v[c] = v[c] + v[d]
v[b] = ror64(v[b] xor v[c], 24)
v[a] = v[a] + v[b] + y
v[d] = ror64(v[d] xor v[a], 16)
v[c] = v[c] + v[d]
v[b] = ror64(v[b] xor v[c], 63)
proc compress(c: var Blake2b, last: int = 0) =
var input, v: array[16, uint64]
for i in 0..15:
input[i] = cast[ptr uint64](addr(c.buffer[i*8]))[]
for i in 0..7:
v[i] = c.hash[i]
v[i+8] = Blake2bIV[i]
v[12] = v[12] xor c.offset[0]
v[13] = v[13] xor c.offset[1]
if (last == 1): v[14] = not(v[14])
for i in 0..11:
G(v, 0, 4, 8, 12, input[Sigma[i][0]], input[Sigma[i][1]])
G(v, 1, 5, 9, 13, input[Sigma[i][2]], input[Sigma[i][3]])
G(v, 2, 6, 10, 14, input[Sigma[i][4]], input[Sigma[i][5]])
G(v, 3, 7, 11, 15, input[Sigma[i][6]], input[Sigma[i][7]])
G(v, 0, 5, 10, 15, input[Sigma[i][8]], input[Sigma[i][9]])
G(v, 1, 6, 11, 12, input[Sigma[i][10]], input[Sigma[i][11]])
G(v, 2, 7, 8, 13, input[Sigma[i][12]], input[Sigma[i][13]])
G(v, 3, 4, 9, 14, input[Sigma[i][14]], input[Sigma[i][15]])
for i in 0..7:
c.hash[i] = c.hash[i] xor v[i] xor v[i+8]
c.buffer_idx = 0
proc blake2b_update*(c: var Blake2b, data: cstring|string|seq|uint8, data_size: int) =
for i in 0..<data_size:
if c.buffer_idx == 128:
inc(c.offset, c.buffer_idx)
compress(c)
when data is cstring or data is string:
c.buffer[c.buffer_idx] = data[i].uint8
elif data is seq:
c.buffer[c.buffer_idx] = data[i]
else:
c.buffer[c.buffer_idx] = data
inc(c.buffer_idx)
proc blake2b_init*(c: var Blake2b, hash_size: uint8,
key: cstring = nil, key_size: int = 0) =
assert(hash_size >= 1'u8 and hash_size <= 64'u8)
assert(key_size >= 0 and key_size <= 64)
c.hash = Blake2bIV
c.hash[0] = c.hash[0] xor 0x01010000 xor cast[uint64](key_size shl 8) xor hash_size
c.hash_size = hash_size
if key_size > 0:
blake2b_update(c, key, key_size)
padding(c.buffer, c.buffer_idx)
c.buffer_idx = 128
proc blake2b_final*(c: var Blake2b): seq[uint8] =
result = newSeq[uint8](c.hash_size)
inc(c.offset, c.buffer_idx)
padding(c.buffer, c.buffer_idx)
compress(c, 1)
for i in 0'u8..<c.hash_size:
result[i.int] = cast[uint8]((c.hash[i div 8] shr (8'u8 * (i and 7)) and 0xFF))
zeroMem(addr(c), sizeof(c))
proc `$`*(d: seq[uint8]): string =
const digits = "0123456789abcdef"
result = ""
for i in 0..high(d):
add(result, digits[(d[i].int shr 4) and 0xF])
add(result, digits[d[i].int and 0xF])
proc getBlake2b*(s: string, hash_size: uint8, key: string = ""): string =
var b: Blake2b
blake2b_init(b, hash_size, cstring(key), len(key))
blake2b_update(b, s, len(s))
result = $blake2b_final(b)
when isMainModule:
import strutils, hex
proc hex2str(s: string): string =
hex.decode s
assert(getBlake2b("abc", 4, "abc") == "b8f97209")
assert(getBlake2b(nil, 4, "abc") == "8ef2d47e")
assert(getBlake2b("abc", 4) == "63906248")
assert(getBlake2b(nil, 4) == "1271cf25")
var b1, b2: Blake2b
blake2b_init(b1, 4)
blake2b_init(b2, 4)
blake2b_update(b1, 97'u8, 1)
blake2b_update(b1, 98'u8, 1)
blake2b_update(b1, 99'u8, 1)
blake2b_update(b2, @[97'u8, 98'u8, 99'u8], 3)
assert($blake2b_final(b1) == $blake2b_final(b2))
let f = open("blake2b-kat.txt", fmRead)
var
data, key, hash, r: string
b: Blake2b
while true:
try:
data = f.readLine()
data = hex2str(data[4.int..data.high])
key = f.readLine()
key = hex2str(key[5..key.high])
hash = f.readLine()
hash = hash[6..hash.high]
r = getBlake2b(data, 64, key)
assert(r == hash)
blake2b_init(b, 64, key, 64)
for i in 0..high(data):
blake2b_update(b, ($data[i]).cstring, 1)
assert($blake2b_final(b) == hash)
discard f.readLine()
except IOError: break
close(f)
echo "ok"

74
src/dagfs/priv/hex.nim Normal file
View File

@ -0,0 +1,74 @@
#[
The MIT License (MIT)
Copyright (c) 2014 Eric S. Bullington
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
]#
proc nibbleFromChar(c: char): int =
case c
of '0'..'9': result = (ord(c) - ord('0'))
of 'a'..'f': result = (ord(c) - ord('a') + 10)
of 'A'..'F': result = (ord(c) - ord('A') + 10)
else:
raise newException(ValueError, "invalid hexadecimal encoding")
proc decode*[T: char|int8|uint8](str: string; result: var openArray[T]) =
assert(result.len == str.len div 2)
for i in 0..<result.len:
result[i] = T((nibbleFromChar(str[2 * i]) shl 4) or nibbleFromChar(str[2 * i + 1]))
proc decode*(str: string): string =
result = newString(len(str) div 2)
decode(str, result)
proc nibbleToChar(nibble: int): char =
const byteMap = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f']
const byteMapLen = len(byteMap)
if nibble < byteMapLen:
return byteMap[nibble];
template encodeTmpl(str: untyped): typed =
let length = (len(str))
result = newString(length * 2)
for i in str.low..str.high:
let a = ord(str[i]) shr 4
let b = ord(str[i]) and ord(0x0f)
result[i * 2] = nibbleToChar(a)
result[i * 2 + 1] = nibbleToChar(b)
proc encode*(bin: string): string =
encodeTmpl(bin)
proc encode*(bin: openarray[char|int8|uint8]): string =
encodeTmpl(bin)
when isMainModule:
assert encode("The sun so bright it leaves no shadows") == "5468652073756e20736f20627269676874206974206c6561766573206e6f20736861646f7773"
const longText = """Man is distinguished, not only by his reason, but by this
singular passion from other animals, which is a lust of the mind,
that by a perseverance of delight in the continued and indefatigable
generation of knowledge, exceeds the short vehemence of any carnal
pleasure."""
assert encode(longText) == "4d616e2069732064697374696e677569736865642c206e6f74206f6e6c792062792068697320726561736f6e2c2062757420627920746869730a2020202073696e67756c61722070617373696f6e2066726f6d206f7468657220616e696d616c732c2077686963682069732061206c757374206f6620746865206d696e642c0a20202020746861742062792061207065727365766572616e6365206f662064656c6967687420696e2074686520636f6e74696e75656420616e6420696e6465666174696761626c650a2020202067656e65726174696f6e206f66206b6e6f776c656467652c2065786365656473207468652073686f727420766568656d656e6365206f6620616e79206361726e616c0a20202020706c6561737572652e"
const tests = ["", "abc", "xyz", "man", "leisure.", "sure.", "erasure.",
"asure.", longText]
for t in items(tests):
assert decode(encode(t)) == t

54
src/dagfs/replicator.nim Normal file
View File

@ -0,0 +1,54 @@
import std/streams, std/strutils, std/os, cbor
import ../dagfs, ./stores
type
DagfsReplicator* = ref DagfsReplicatorObj
DagfsReplicatorObj* = object of DagfsStoreObj
toStore, fromStore: DagfsStore
cache: string
cacheCid: Cid
proc replicatedPut(s: DagfsStore; blk: string): Cid =
var r = DagfsReplicator(s)
r.toStore.put blk
proc replicatedGetBuffer(s: DagfsStore; cid: Cid; buf: pointer; len: Natural): int =
var r = DagfsReplicator