CIDv0 support
This commit is contained in:
parent
3d0bbca3b4
commit
7203eb2a65
64
README.md
64
README.md
|
|
@ -1,60 +1,8 @@
|
|||
# WARNING
|
||||
An IPFS library. No support for Protocol Buffers, which means no support
|
||||
for the current IPFS file-system.
|
||||
|
||||
Contains my own contrived standards and formats that will break often.
|
||||
Upstream development on CBOR file-system structures has predictably stalled
|
||||
over discussion of features such as non-ASCII, non-unicode file-names
|
||||
and extended attributes.
|
||||
|
||||
## ipldrepl
|
||||
|
||||
A Lisp REPL utility for storing files and directories in IPLD.
|
||||
|
||||
### Functions
|
||||
|
||||
#### `(apply <function> <list>)`
|
||||
|
||||
Standard Lisp `apply` function, apply a list as arguments to a function.
|
||||
|
||||
#### `(cbor <cid>)`
|
||||
|
||||
Return CBOR encoding of UnixFS node as a diagnostic string.
|
||||
Provided for illustrating canonicalized CBOR encoding.
|
||||
|
||||
#### `(cons <head> <tail>)`
|
||||
|
||||
Standard Lisp `cons` function, prepend to a list.
|
||||
|
||||
#### `(copy <cid> <from> <to>)`
|
||||
|
||||
Duplicate a directory entry.
|
||||
|
||||
#### `(define <symbol> <value>)`
|
||||
|
||||
Bind a value to a symbol. Returns value.
|
||||
|
||||
#### `(glob <glob-string> <...>)`
|
||||
|
||||
Return a list of paths matching a Unix-style glob string.
|
||||
|
||||
#### `(ingest <path> <...>)`
|
||||
|
||||
Ingest a list of paths to the store, returning a CID to a directory with
|
||||
the contents each path under the trailing name of the path.
|
||||
|
||||
#### `(map <function> <list>)`
|
||||
|
||||
Standard Lisp `map`, apply a function to each member of a list.
|
||||
|
||||
#### `(merge <cid> <...>)`
|
||||
|
||||
Merge a list of root directories represented by CIDs into a new root.
|
||||
Members of the root are not merged recursively.
|
||||
|
||||
#### `(path <string>)`
|
||||
|
||||
Convert a string to a path, if the path is valid and present.
|
||||
|
||||
#### `(root <string> <cid>)`
|
||||
|
||||
Create a new root contaning the giving CID at the given name.
|
||||
|
||||
#### `(walk <cid> <string>)`
|
||||
|
||||
Walk a path down one CID to another.
|
||||
Current plan is to fork, postponed indefinitly.
|
||||
|
|
|
|||
1025
blake2b-kat.txt
1025
blake2b-kat.txt
File diff suppressed because it is too large
Load Diff
|
|
@ -2,11 +2,12 @@
|
|||
|
||||
version = "0.1.1"
|
||||
author = "Emery Hemingway"
|
||||
description = "IPLD library"
|
||||
description = "InterPlanetary Linked Data library"
|
||||
license = "GPLv3"
|
||||
srcDir = "src"
|
||||
|
||||
# Dependencies
|
||||
|
||||
requires "nim >= 0.17.3", "nimSHA2", "base58", "cbor >= 0.2.0"
|
||||
requires "nim >= 0.18.1", "nimSHA2", "base58", "cbor >= 0.2.0"
|
||||
|
||||
bin = @["ipld/ipldrepl","ipld/ipldcat"]
|
||||
bin = @["src/ipld/ipldrepl","src/ipld/ipldcat"]
|
||||
|
|
|
|||
170
ipld/ipld.nim
170
ipld/ipld.nim
|
|
@ -1,170 +0,0 @@
|
|||
import nimSHA2, streams, multiformats, base58.bitcoin, cbor, hex, hashes, blake2
|
||||
|
||||
const MaxBlockSize* = 1 shl 18
|
||||
## Maximum supported block size.
|
||||
|
||||
type Cid* = object
|
||||
## Content IDentifier, used to identify blocks.
|
||||
digest*: seq[uint8]
|
||||
hash*: MulticodecTag
|
||||
codec*: MulticodecTag
|
||||
ver*: int
|
||||
|
||||
proc initCid*(): Cid =
|
||||
## Initialize an invalid CID.
|
||||
Cid(hash: MulticodecTag.Invalid, codec: MulticodecTag.Invalid)
|
||||
|
||||
proc isValid*(x: Cid): bool =
|
||||
## Check that a CID has been properly initialized.
|
||||
x.hash != MulticodecTag.Invalid
|
||||
|
||||
proc `==`*(x, y: Cid): bool =
|
||||
## Compare two CIDs. Must be of the same type and
|
||||
## use the same hash algorithm.
|
||||
result =
|
||||
x.ver == y.ver and
|
||||
x.codec == y.codec and
|
||||
x.hash == y.hash and
|
||||
x.digest == y.digest
|
||||
|
||||
proc hash*(cid: Cid): Hash =
|
||||
## Reduce a CID into an integer for use in tables.
|
||||
result = hash cid.digest
|
||||
result = result !& cid.ver !& cid.codec.int !& cid.hash.int
|
||||
result = !$result
|
||||
|
||||
proc isRaw*(cid: Cid): bool =
|
||||
## Test if a CID represents a raw block.
|
||||
cid.codec == MulticodecTag.Raw
|
||||
|
||||
proc isDagCbor*(cid: Cid): bool =
|
||||
## Test if a CID represents CBOR encoded data.
|
||||
cid.codec == MulticodecTag.DagCbor
|
||||
|
||||
proc toBin(cid: Cid): string =
|
||||
let s = newStringStream()
|
||||
s.writeUvarint cid.ver
|
||||
s.writeUvarint cid.codec.int
|
||||
s.writeUvarint cid.hash.int
|
||||
s.writeUvarint cid.digest.len
|
||||
for e in cid.digest:
|
||||
s.write e
|
||||
s.setPosition 0
|
||||
result = s.readAll
|
||||
close s
|
||||
|
||||
proc toRaw*(cid: Cid): string =
|
||||
## Return CID encoded in binary.
|
||||
MultibaseTag.Identity.char & cid.toBIn
|
||||
|
||||
proc newCborBytes*(cid: Cid): CborNode = newCborBytes cid.toRaw
|
||||
## Generate a CBOR representation of a CID.
|
||||
|
||||
proc toHex*(cid: Cid): string =
|
||||
## Return CID encoded in hexidecimal.
|
||||
MultibaseTag.Base16.char & hex.encode(cid.toBin)
|
||||
|
||||
proc toBase58*(cid: Cid): string =
|
||||
## Return CID encoded in base 58.
|
||||
MultibaseTag.Base58btc.char & bitcoin.encode(cid.toBin)
|
||||
|
||||
proc `$`*(cid: Cid): string = cid.toBase58
|
||||
## Return CID in base 58, the default textual encoding.
|
||||
|
||||
proc parseCid*(s: string): Cid =
|
||||
## Detect CID encoding and parse from a string.
|
||||
if unlikely(s.len < (1+1+1+1)):
|
||||
raise newException(ValueError, "too short to be a valid CID")
|
||||
var
|
||||
raw: string
|
||||
off: int
|
||||
codec, hash: int
|
||||
digestLen: int
|
||||
case s[0].MultibaseTag
|
||||
of MultibaseTag.Identity:
|
||||
raw = s
|
||||
off = 1
|
||||
of MultibaseTag.Base16, MultibaseTag.InconsistentBase16:
|
||||
raw = hex.decode(s[1..s.high])
|
||||
if unlikely(raw.isNil):
|
||||
raise newException(ValueError, "not a CID")
|
||||
of MultibaseTag.Base58btc:
|
||||
raw = bitcoin.decode(s[1..s.high])
|
||||
else:
|
||||
raise newException(ValueError, "unknown multibase encoding tag")
|
||||
off.inc parseUvarint(raw, result.ver, off)
|
||||
off.inc parseUvarint(raw, codec, off)
|
||||
off.inc parseUvarint(raw, hash, off)
|
||||
off.inc parseUvarint(raw, digestLen, off)
|
||||
if unlikely(off + digestLen != raw.len):
|
||||
raise newException(ValueError, "invalid multihash length")
|
||||
result.digest = newSeq[uint8](digestLen)
|
||||
for i in 0..<digestLen:
|
||||
result.digest[i] = (uint8)raw[i+off]
|
||||
result.hash = hash.MulticodecTag
|
||||
result.codec = codec.MulticodecTag
|
||||
|
||||
proc CidSha256*(data: string; codec = MulticodecTag.Raw): Cid =
|
||||
## Generate a CID for a string of data using the SHA 256 hash algorithm.
|
||||
let sha = computeSHA256(data)
|
||||
result.digest = newSeq[uint8](32)
|
||||
for i in 0..31:
|
||||
result.digest[i] = (uint8)sha[i]
|
||||
result.hash = MulticodecTag.Sha2_256
|
||||
result.codec = codec
|
||||
result.ver = 1
|
||||
|
||||
proc CidBlake2b256*(data: string; codec = MulticodecTag.Raw): Cid =
|
||||
## Generate a CID for a string of data using the BLAKE2b hash algorithm.
|
||||
var b: Blake2b
|
||||
blake2b_init(b, 32, nil, 0)
|
||||
blake2b_update(b, data, data.len)
|
||||
|
||||
result.digest = blake2b_final(b)
|
||||
result.hash = MulticodecTag.Blake2b_256
|
||||
result.codec = codec
|
||||
result.ver = 1
|
||||
|
||||
proc verify*(cid: Cid; data: string): bool =
|
||||
## Verify that a string of data corresponds to a CID.
|
||||
case cid.hash
|
||||
of MulticodecTag.Sha2_256:
|
||||
let sha = computeSHA256(data)
|
||||
for i in 0..31:
|
||||
if cid.digest[i] != (uint8)sha[i]:
|
||||
return false
|
||||
result = true
|
||||
of MulticodecTag.Blake2b_256:
|
||||
var b: Blake2b
|
||||
blake2b_init(b, 32, nil, 0)
|
||||
blake2b_update(b, data, data.len)
|
||||
let digest = blake2b_final(b)
|
||||
result = (cid.digest == digest)
|
||||
else:
|
||||
raise newException(ValueError, "unknown hash type " & $cid.hash)
|
||||
|
||||
iterator simpleChunks*(s: Stream; size = 256 * 1024): string =
|
||||
## Iterator that breaks a stream into simple chunks.
|
||||
while not s.atEnd:
|
||||
yield s.readStr size
|
||||
|
||||
when isMainModule:
|
||||
import times
|
||||
|
||||
let data = newString MaxBlockSize
|
||||
block sha256:
|
||||
var i = 0
|
||||
let t0 = cpuTime()
|
||||
while i < 100:
|
||||
discard CidSha256 data
|
||||
inc i
|
||||
let d = cpuTime() - t0
|
||||
echo "SHA256 ticks: ", d
|
||||
block blake2b:
|
||||
var i = 0
|
||||
let t0 = cpuTime()
|
||||
while i < 100:
|
||||
discard CidBlake2b256 data
|
||||
inc i
|
||||
let d = cpuTime() - t0
|
||||
echo "BLAKE2b ticks: ", d
|
||||
|
|
@ -0,0 +1,235 @@
|
|||
import std/hashes, std/streams, std/strutils
|
||||
|
||||
import nimSHA2, ./ipld/multiformats, base58/bitcoin, cbor, ./ipld/priv/hex, ./ipld/priv/blake2
|
||||
|
||||
const MaxBlockSize* = 1 shl 18
|
||||
## Maximum supported block size.
|
||||
|
||||
type
|
||||
CidVersion* = enum CIDv0, CIDv1
|
||||
|
||||
Cid* = object
|
||||
## Content IDentifier, used to identify blocks.
|
||||
case kind*: CidVersion
|
||||
of CIDv0:
|
||||
sha256: array[32, uint8]
|
||||
of CIDv1:
|
||||
digest*: seq[uint8] # this is stupid, make it a fixed size
|
||||
hash*: MulticodecTag
|
||||
codec*: MulticodecTag
|
||||
ver*: int
|
||||
|
||||
proc initCid*(): Cid =
|
||||
## Initialize an invalid CID.
|
||||
Cid(kind: CIDv1, hash: MulticodecTag.Invalid, codec: MulticodecTag.Invalid)
|
||||
|
||||
proc isValid*(x: Cid): bool =
|
||||
## Check that a CID has been properly initialized.
|
||||
case x.kind
|
||||
of CIDv0: true # whatever
|
||||
of CIDv1: x.hash != MulticodecTag.Invalid
|
||||
|
||||
proc `==`*(x, y: Cid): bool =
|
||||
## Compare two CIDs. Must be of the same type and
|
||||
## use the same hash algorithm.
|
||||
if x.kind == y.kind:
|
||||
case x.kind
|
||||
of CIDv0:
|
||||
result = (x.sha256 == y.sha256)
|
||||
of CIDv1:
|
||||
result = (
|
||||
x.ver == y.ver and
|
||||
x.codec == y.codec and
|
||||
x.hash == y.hash and
|
||||
x.digest == y.digest)
|
||||
|
||||
proc hash*(cid: Cid): Hash =
|
||||
## Reduce a CID into an integer for use in tables.
|
||||
case cid.kind
|
||||
of CIDv0:
|
||||
result = hash cid.sha256
|
||||
of CIDv1:
|
||||
result = hash cid.digest
|
||||
result = result !& cid.ver !& cid.codec.int !& cid.hash.int
|
||||
result = !$result
|
||||
|
||||
proc isRaw*(cid: Cid): bool =
|
||||
## Test if a CID represents a raw block.
|
||||
cid.kind == CIDv1 and cid.codec == MulticodecTag.Raw
|
||||
|
||||
proc isDag*(cid: Cid): bool =
|
||||
## Test if a CID represents protobuf or CBOR encoded data.
|
||||
cid.kind == CIDv0 or cid.codec in {MulticodecTag.DagPb, MulticodecTag.DagCbor}
|
||||
|
||||
proc isDagPb*(cid: Cid): bool =
|
||||
## Test if a CID represents protobuf encoded data.
|
||||
cid.kind == CIDv0 or cid.codec == MulticodecTag.DagPb
|
||||
|
||||
proc isDagCbor*(cid: Cid): bool =
|
||||
## Test if a CID represents CBOR encoded data.
|
||||
cid.kind == CIDv1 and cid.codec == MulticodecTag.DagCbor
|
||||
|
||||
proc toBin(cid: Cid): string =
|
||||
case cid.kind
|
||||
of CIDv0:
|
||||
result = newString(34)
|
||||
result[0] = 0x12.char
|
||||
result[1] = 0x20.char
|
||||
var sha = cid.sha256
|
||||
copyMem(result[2].addr, sha[0].addr, 32)
|
||||
of CIDv1:
|
||||
let s = newStringStream()
|
||||
s.writeUvarint cid.ver
|
||||
s.writeUvarint cid.codec.int
|
||||
s.writeUvarint cid.hash.int
|
||||
s.writeUvarint cid.digest.len
|
||||
for e in cid.digest:
|
||||
s.write e
|
||||
s.setPosition 0
|
||||
result = s.readAll
|
||||
close s
|
||||
|
||||
proc toRaw*(cid: Cid): string =
|
||||
## Return CID encoded in binary.
|
||||
case cid.kind
|
||||
of CIDv0:
|
||||
cid.toBin
|
||||
of CIDv1:
|
||||
MultibaseTag.Identity.char & cid.toBIn
|
||||
|
||||
proc newCborBytes*(cid: Cid): CborNode = newCborBytes cid.toRaw
|
||||
## Generate a CBOR representation of a CID.
|
||||
|
||||
proc toHex*(cid: Cid): string =
|
||||
## Return CID encoded in hexidecimal.
|
||||
assert(isValid cid)
|
||||
MultibaseTag.Base16.char & hex.encode(cid.toBin)
|
||||
|
||||
proc toBase58*(cid: Cid): string =
|
||||
## Return CID encoded in base 58.
|
||||
assert(isValid cid)
|
||||
case cid.kind
|
||||
of CIDv0:
|
||||
bitcoin.encode(cid.toBin)
|
||||
of CIDv1:
|
||||
MultibaseTag.Base58btc.char & bitcoin.encode(cid.toBin)
|
||||
|
||||
proc `$`*(cid: Cid): string =
|
||||
## Return CID in base 58, the default textual encoding.
|
||||
cid.toBase58
|
||||
|
||||
proc parseCid*(s: string): Cid =
|
||||
## Detect CID encoding and parse from a string.
|
||||
if unlikely(s.len < (1+1+1+1)):
|
||||
raise newException(ValueError, "too short to be a valid CID")
|
||||
var
|
||||
raw: string
|
||||
off: int
|
||||
codec, hash: int
|
||||
digestLen: int
|
||||
if s.len == 46 and s.startsWith "Qm":
|
||||
var data = bitcoin.decode(s)
|
||||
if data.len == 34 and data.startsWith "\x12\x20":
|
||||
result.kind = CIDv0
|
||||
copyMem(result.sha256[0].addr, data[2].addr, 32)
|
||||
else:
|
||||
raise newException(ValueError, "invalid CIDv0")
|
||||
else:
|
||||
case s[0].MultibaseTag
|
||||
of MultibaseTag.Identity:
|
||||
raw = s
|
||||
off = 1
|
||||
of MultibaseTag.Base16, MultibaseTag.InconsistentBase16:
|
||||
raw = hex.decode(s[1..s.high])
|
||||
if unlikely(raw.isNil):
|
||||
raise newException(ValueError, "not a CID")
|
||||
of MultibaseTag.Base58btc:
|
||||
raw = bitcoin.decode(s[1..s.high])
|
||||
else:
|
||||
raise newException(ValueError, "unknown multibase encoding tag")
|
||||
off.inc parseUvarint(raw, result.ver, off)
|
||||
off.inc parseUvarint(raw, codec, off)
|
||||
off.inc parseUvarint(raw, hash, off)
|
||||
off.inc parseUvarint(raw, digestLen, off)
|
||||
if unlikely(off + digestLen != raw.len):
|
||||
raise newException(ValueError, "invalid multihash length")
|
||||
result.kind = CIDv0
|
||||
result.digest = newSeq[uint8](digestLen)
|
||||
for i in 0..<digestLen:
|
||||
result.digest[i] = (uint8)raw[i+off]
|
||||
result.hash = hash.MulticodecTag
|
||||
result.codec = codec.MulticodecTag
|
||||
|
||||
proc CidSha256*(data: string; codec = MulticodecTag.Raw): Cid =
|
||||
## Generate a CID for a string of data using the SHA 256 hash algorithm.
|
||||
result.kind = CIDv1
|
||||
let sha = computeSHA256(data)
|
||||
result.digest = newSeq[uint8](32)
|
||||
for i in 0..31:
|
||||
result.digest[i] = (uint8)sha[i]
|
||||
result.hash = MulticodecTag.Sha2_256
|
||||
result.codec = codec
|
||||
result.ver = 1
|
||||
|
||||
proc CidBlake2b256*(data: string; codec = MulticodecTag.Raw): Cid =
|
||||
## Generate a CID for a string of data using the BLAKE2b hash algorithm.
|
||||
result.kind = CIDv1
|
||||
var b: Blake2b
|
||||
blake2b_init(b, 32, nil, 0)
|
||||
blake2b_update(b, data, data.len)
|
||||
result.digest = blake2b_final(b)
|
||||
result.hash = MulticodecTag.Blake2b_256
|
||||
result.codec = codec
|
||||
result.ver = 1
|
||||
|
||||
proc verify*(cid: Cid; data: string): bool =
|
||||
## Verify that a string of data corresponds to a CID.
|
||||
case cid.kind
|
||||
of CIDv0:
|
||||
let sha = computeSHA256(data)
|
||||
for i in 0..31:
|
||||
if cid.sha256[i] != (uint8)sha[i]:
|
||||
return false
|
||||
result = true
|
||||
of CIDv1:
|
||||
case cid.hash
|
||||
of MulticodecTag.Sha2_256:
|
||||
let sha = computeSHA256(data)
|
||||
for i in 0..31:
|
||||
if cid.digest[i] != (uint8)sha[i]:
|
||||
return false
|
||||
result = true
|
||||
of MulticodecTag.Blake2b_256:
|
||||
var b: Blake2b
|
||||
blake2b_init(b, 32, nil, 0)
|
||||
blake2b_update(b, data, data.len)
|
||||
let digest = blake2b_final(b)
|
||||
result = (cid.digest == digest)
|
||||
else:
|
||||
raise newException(ValueError, "unknown hash type " & $cid.hash)
|
||||
|
||||
iterator simpleChunks*(s: Stream; size = 256 * 1024): string =
|
||||
## Iterator that breaks a stream into simple chunks.
|
||||
while not s.atEnd:
|
||||
yield s.readStr size
|
||||
|
||||
when isMainModule:
|
||||
import times
|
||||
|
||||
let data = newString MaxBlockSize
|
||||
block sha256:
|
||||
var i = 0
|
||||
let t0 = cpuTime()
|
||||
while i < 100:
|
||||
discard CidSha256 data
|
||||
inc i
|
||||
let d = cpuTime() - t0
|
||||
echo "SHA256 ticks: ", d
|
||||
block blake2b:
|
||||
var i = 0
|
||||
let t0 = cpuTime()
|
||||
while i < 100:
|
||||
discard CidBlake2b256 data
|
||||
inc i
|
||||
let d = cpuTime() - t0
|
||||
echo "BLAKE2b ticks: ", d
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
/*
|
||||
* \brief C++ base of IPLD client
|
||||
* \author Emery Hemingway
|
||||
* \date 2017-11-08
|
||||
*/
|
||||
|
||||
/*
|
||||
* Copyright (C) 2017 Genode Labs GmbH
|
||||
*
|
||||
* This file is part of the Genode OS framework, which is distributed
|
||||
* under the terms of the GNU Affero General Public License version 3.
|
||||
*/
|
||||
|
||||
/* Genode includes */
|
||||
#include <ipld_session/connection.h>
|
||||
#include <base/heap.h>
|
||||
|
||||
struct IpldClientBase
|
||||
{
|
||||
Genode::Heap heap;
|
||||
Genode::Allocator_avl tx_packet_alloc { &heap };
|
||||
Ipld::Connection conn;
|
||||
|
||||
IpldClientBase(Genode::Env *env, char const *label, Genode::size_t tx_buf_size)
|
||||
: heap(env->pd(), env->rm()),
|
||||
conn(*env, tx_packet_alloc, label, tx_buf_size)
|
||||
{ }
|
||||
};
|
||||
|
|
@ -0,0 +1,147 @@
|
|||
#
|
||||
# \brief IpldStore interface to the IPLD session
|
||||
# \author Emery Hemingway
|
||||
# \date 2017-11-04
|
||||
#
|
||||
|
||||
#
|
||||
# Copyright (C) 2017 Genode Labs GmbH
|
||||
#
|
||||
# This file is part of the Genode OS framework, which is distributed
|
||||
# under the terms of the GNU Affero General Public License version 3.
|
||||
#
|
||||
|
||||
when not defined(genode):
|
||||
{.error: "Genode only IPLD client".}
|
||||
|
||||
import cbor, genode, tables, strutils
|
||||
|
||||
import ipld, ipld/multiformats, ipld/store, ipld/genode/ipldsession
|
||||
|
||||
const
|
||||
currentPath = currentSourcePath.rsplit("/", 1)[0]
|
||||
ipldClientH = currentPath & "/ipldclient.h"
|
||||
{.passC: "-I" & currentPath.}
|
||||
|
||||
type
|
||||
IpldClientBase {.importcpp, header: ipldClientH.} = object
|
||||
IpldClientCpp = Constructible[IpldClientBase]
|
||||
|
||||
proc sigh_ack_avail(cpp: IpldClientCpp; sig: SignalContextCapability) {.
|
||||
importcpp: "#->conn.channel().sigh_ack_avail(@)", tags: [RpcEffect].}
|
||||
|
||||
proc readyToSubmit(cpp: IpldClientCpp): bool {.
|
||||
importcpp: "#->conn.source().ready_to_submit()".}
|
||||
|
||||
proc readyToAck(cpp: IpldClientCpp): bool {.
|
||||
importcpp: "#->conn.source().ready_to_ack()".}
|
||||
|
||||
proc ackAvail(cpp: IpldClientCpp): bool {.
|
||||
importcpp: "#->conn.source().ack_avail()".}
|
||||
|
||||
proc allocPacket(cpp: IpldClientCpp; size = MaxPacketSize): IpldPacket {.
|
||||
importcpp: "#->conn.source().alloc_packet(@)".}
|
||||
|
||||
proc packetContent(cpp: IpldClientCpp; pkt: IpldPacket): pointer {.
|
||||
importcpp: "#->conn.source().packet_content(@)".}
|
||||
|
||||
proc submitPacket(cpp: IpldClientCpp; pkt: IpldPacket; cid: cstring; op: IpldOpcode) {.
|
||||
importcpp: "#->conn.source().submit_packet(Ipld::Packet(#, (char const *)#, #))".}
|
||||
|
||||
proc getAckedPacket(cpp: IpldClientCpp): IpldPacket {.
|
||||
importcpp: "#->conn.source().get_acked_packet()".}
|
||||
|
||||
proc releasePacket(cpp: IpldClientCpp; pkt: IpldPacket) {.
|
||||
importcpp: "#->conn.source().release_packet(@)".}
|
||||
|
||||
type
|
||||
IpldClient* = ref IpldClientObj
|
||||
IpldClientObj = object of IpldStoreObj
|
||||
## IPLD session client
|
||||
cpp: IpldClientCpp
|
||||
|
||||
proc icClose(s: IpldStore) =
|
||||
var ic = IpldClient(s)
|
||||
destruct ic.cpp
|
||||
|
||||
proc icPut(s: IpldStore; blk: string; hash: MulticodecTag): Cid =
|
||||
## Put block to Ipld server, blocks for two packet round-trip.
|
||||
let ic = IpldClient(s)
|
||||
var blk = blk
|
||||
var pktCid = initCid()
|
||||
pktCid.hash = hash
|
||||
assert(ic.cpp.readyToSubmit, "Ipld client packet queue congested")
|
||||
var pkt = ic.cpp.allocPacket(blk.len)
|
||||
let pktBuf = ic.cpp.packetContent pkt
|
||||
defer: ic.cpp.releasePacket pkt
|
||||
assert(not pktBuf.isNil, "allocated packet has nil content")
|
||||
assert(pkt.size >= blk.len)
|
||||
pkt.setLen blk.len
|
||||
copyMem(pktBuf, blk[0].addr, blk.len)
|
||||
assert(ic.cpp.readyToSubmit, "Ipld client packet queue congested")
|
||||
ic.cpp.submitPacket(pkt, pktCid.toHex, PUT)
|
||||
let ack = ic.cpp.getAckedPacket()
|
||||
doAssert(ack.error == OK)
|
||||
result = ack.cid()
|
||||
assert(result.isValid, "server returned a packet with and invalid CID")
|
||||
|
||||
proc icGetBuffer(s: IpldStore; cid: Cid; buf: pointer; len: Natural): int =
|
||||
## Get from Ipld server, blocks for packet round-trip.
|
||||
let ic = IpldClient(s)
|
||||
assert(ic.cpp.readyToSubmit, "Ipld client packet queue congested")
|
||||
let pkt = ic.cpp.allocPacket len
|
||||
ic.cpp.submitPacket(pkt, cid.toHex, GET)
|
||||
let ack = ic.cpp.getAckedPacket
|
||||
doAssert(ack.cid == cid)
|
||||
if ack.error == OK:
|
||||
let pktBuf = ic.cpp.packetContent ack
|
||||
assert(not pktBuf.isNil, "ack packet has nil content")
|
||||
assert(ack.len <= len)
|
||||
assert(ack.len > 0)
|
||||
result = ack.len
|
||||
copyMem(buf, pktBuf, result)
|
||||
if pkt.size > 0:
|
||||
ic.cpp.releasePacket pkt
|
||||
# free the original packet that was allocated
|
||||
case ack.error:
|
||||
of OK: discard
|
||||
of MISSING:
|
||||
raise cid.newMissingObject
|
||||
else:
|
||||
raise newException(SystemError, "Ipld packet error " & $ack.error)
|
||||
|
||||
proc icGet(s: IpldStore; cid: Cid; result: var string) =
|
||||
## Get from Ipld server, blocks for packet round-trip.
|
||||
let ic = IpldClient(s)
|
||||
assert(ic.cpp.readyToSubmit, "Ipld client packet queue congested")
|
||||
let pkt = ic.cpp.allocPacket()
|
||||
defer: ic.cpp.releasePacket pkt
|
||||
ic.cpp.submitPacket(pkt, cid.toHex, GET)
|
||||
let ack = ic.cpp.getAckedPacket()
|
||||
doAssert(ack.cid == cid)
|
||||
case ack.error:
|
||||
of OK:
|
||||
let ackBuf = ic.cpp.packetContent ack
|
||||
assert(not ackBuf.isNil)
|
||||
assert(ack.len > 0)
|
||||
result.setLen ack.len
|
||||
copyMem(result[0].addr, ackBuf, result.len)
|
||||
assert(cid.verify(result), "Ipld client packet failed verification")
|
||||
of MISSING:
|
||||
raise cid.newMissingObject
|
||||
else:
|
||||
raise newException(SystemError, "Ipld packet error " & $ack.error)
|
||||
|
||||
const
|
||||
DefaultIpldBufferSize* = 1 shl 20
|
||||
|
||||
proc newIpldClient*(env: GenodeEnv; label = ""; bufferSize = DefaultIpldBufferSize): IpldClient =
|
||||
## Blocks retrieved by `get` are not verified.
|
||||
proc construct(cpp: IpldClientCpp; env: GenodeEnv; label: cstring; txBufSize: int) {.
|
||||
importcpp.}
|
||||
new result
|
||||
construct(result.cpp, env, label, bufferSize)
|
||||
result.closeImpl = icClose
|
||||
result.putImpl = icPut
|
||||
result.getBufferImpl = icGetBuffer
|
||||
result.getImpl = icGet
|
||||
|
|
@ -0,0 +1,56 @@
|
|||
/*
|
||||
* \brief IPLD C++ session component
|
||||
* \author Emery Hemingway
|
||||
* \date 2017-11-07
|
||||
*/
|
||||
|
||||
/*
|
||||
* Copyright (C) 2017 Genode Labs GmbH
|
||||
*
|
||||
* This file is part of the Genode OS framework, which is distributed
|
||||
* under the terms of the GNU Affero General Public License version 3.
|
||||
*/
|
||||
|
||||
#ifndef _INCLUDE__NIM__IPLDSERVER_H_
|
||||
#define _INCLUDE__NIM__IPLDSERVER_H_
|
||||
|
||||
#include <ipld_session/rpc_object.h>
|
||||
#include <base/heap.h>
|
||||
#include <base/attached_ram_dataspace.h>
|
||||
|
||||
struct Communication_buffer
|
||||
{
|
||||
Genode::Attached_ram_dataspace _tx_ds;
|
||||
|
||||
Communication_buffer(Genode::Pd_session &pd,
|
||||
Genode::Region_map &rm,
|
||||
Genode::size_t tx_buf_size)
|
||||
: _tx_ds(pd, rm, tx_buf_size) { }
|
||||
};
|
||||
|
||||
struct IpldSessionComponentBase : Communication_buffer,
|
||||
Ipld::Session_rpc_object
|
||||
{
|
||||
static Genode::size_t tx_buf_size(char const *args)
|
||||
{
|
||||
Genode::size_t const buf_size = Genode::Arg_string::find_arg(
|
||||
args, "tx_buf_size").aligned_size();
|
||||
if (!buf_size)
|
||||
throw Genode::Service_denied();
|
||||
return buf_size;
|
||||
}
|
||||
|
||||
IpldSessionComponentBase(Genode::Env *env, char const *args)
|
||||
:
|
||||
Communication_buffer(env->pd(), env->rm(), tx_buf_size(args)),
|
||||
Session_rpc_object(env->rm(), env->ep().rpc_ep(), _tx_ds.cap())
|
||||
{ }
|
||||
|
||||
void packetHandler(Genode::Signal_context_capability cap)
|
||||
{
|
||||
_tx.sigh_ready_to_ack(cap);
|
||||
_tx.sigh_packet_avail(cap);
|
||||
}
|
||||
};
|
||||
|
||||
#endif /* _INCLUDE__NIM__IPLDSERVER_H_ */
|
||||
|
|
@ -0,0 +1,142 @@
|
|||
#
|
||||
# \brief IPLD server factory
|
||||
# \author Emery Hemingway
|
||||
# \date 2017-11-11
|
||||
#
|
||||
|
||||
#
|
||||
# Copyright (C) 2017 Genode Labs GmbH
|
||||
#
|
||||
# This file is part of the Genode OS framework, which is distributed
|
||||
# under the terms of the GNU Affero General Public License version 3.
|
||||
#
|
||||
|
||||
import std/strtabs, std/tables, std/xmltree, std/strutils
|
||||
|
||||
import cbor, genode, genode/signals, genode/servers, ipld, ipld/store, ipldsession
|
||||
|
||||
const
|
||||
currentPath = currentSourcePath.rsplit("/", 1)[0]
|
||||
ipldserverH = currentPath & "/ipldserver.h"
|
||||
|
||||
type
|
||||
IpldSessionComponentBase {.importcpp, header: ipldserverH.} = object
|
||||
SessionCpp = Constructible[IpldSessionComponentBase]
|
||||
Session = ref object
|
||||
cpp: SessionCpp
|
||||
sig: SignalHandler
|
||||
store: IpldStore
|
||||
id: SessionId
|
||||
label: string
|
||||
|
||||
proc processPacket(session: Session; pkt: var IpldPacket) =
|
||||
proc packetContent(cpp: SessionCpp; pkt: IpldPacket): pointer {.
|
||||
importcpp: "#->sink().packet_content(@)".}
|
||||
let cid = pkt.cid
|
||||
case pkt.operation
|
||||
of PUT:
|
||||
try:
|
||||
var
|
||||
pktBuf = session.cpp.packetContent pkt
|
||||
heapBuf = newString pkt.len
|
||||
copyMem(heapBuf[0].addr, pktBuf, heapBuf.len)
|
||||
let putCid = session.store.put(heapBuf, cid.hash)
|
||||
assert(putCid.isValid, "server packet returned invalid CID from put")
|
||||
pkt.setCid putCid
|
||||
except:
|
||||
echo "unhandled PUT error ", getCurrentExceptionMsg()
|
||||
pkt.setError ERROR
|
||||
of GET:
|
||||
try:
|
||||
let
|
||||
pktBuf = session.cpp.packetContent pkt
|
||||
n = session.store.getBuffer(cid, pktBuf, pkt.size)
|
||||
pkt.setLen n
|
||||
except BufferTooSmall:
|
||||
pkt.setError OVERSIZE
|
||||
except MissingObject:
|
||||
pkt.setError MISSING
|
||||
except:
|
||||
echo "unhandled GET error ", getCurrentExceptionMsg()
|
||||
pkt.setError ERROR
|
||||
else:
|
||||
echo "invalid packet operation"
|
||||
pkt.setError ERROR
|
||||
|
||||
proc newSession(env: GenodeEnv; store: IpldStore; id: SessionId; label, args: string): Session =
|
||||
## Create a new session and packet handling procedure
|
||||
let session = new Session
|
||||
assert(not session.isNil)
|
||||
proc construct(cpp: SessionCpp; env: GenodeEnv; args: cstring) {.importcpp.}
|
||||
session.cpp.construct(env, args)
|
||||
session.store = store
|
||||
session.id = id
|
||||
session.label = label
|
||||
session.sig = env.ep.newSignalHandler do ():
|
||||
proc packetAvail(cpp: SessionCpp): bool {.
|
||||
importcpp: "#->sink().packet_avail()".}
|
||||
proc readyToAck(cpp: SessionCpp): bool {.
|
||||
importcpp: "#->sink().ready_to_ack()".}
|
||||
while session.cpp.packetAvail and session.cpp.readyToAck:
|
||||
proc getPacket(cpp: SessionCpp): IpldPacket {.
|
||||
importcpp: "#->sink().get_packet()".}
|
||||
var pkt = session.cpp.getPacket()
|
||||
session.processPacket pkt
|
||||
proc acknowledgePacket(cpp: SessionCpp; pkt: IpldPacket) {.
|
||||
importcpp: "#->sink().acknowledge_packet(@)".}
|
||||
session.cpp.acknowledgePacket(pkt)
|
||||
|
||||
proc packetHandler(cpp: SessionCpp; cap: SignalContextCapability) {.
|
||||
importcpp: "#->packetHandler(@)".}
|
||||
session.cpp.packetHandler(session.sig.cap)
|
||||
result = session
|
||||
|
||||
proc manage(ep: Entrypoint; s: Session): IpldSessionCapability =
|
||||
## Manage a session from the default entrypoint.
|
||||
proc manage(ep: Entrypoint; cpp: SessionCpp): IpldSessionCapability {.
|
||||
importcpp: "#.manage(*#)".}
|
||||
result = ep.manage(s.cpp)
|
||||
GC_ref s
|
||||
|
||||
proc dissolve(ep: Entrypoint; s: Session) =
|
||||
## Dissolve a session from the entrypoint so that it can be freed.
|
||||
proc dissolve(ep: Entrypoint; cpp: SessionCpp) {.
|
||||
importcpp: "#.dissolve(*#)".}
|
||||
ep.dissolve(s.cpp)
|
||||
destruct(s.cpp)
|
||||
dissolve(s.sig)
|
||||
GC_unref s
|
||||
|
||||
type
|
||||
IpldServer* = ref object
|
||||
env: GenodeEnv
|
||||
store*: IpldStore
|
||||
sessions*: Table[SessionId, Session]
|
||||
|
||||
proc newIpldServer*(env: GenodeEnv; store: IpldStore): IpldServer =
|
||||
IpldServer(
|
||||
env: env, store: store,
|
||||
sessions: initTable[SessionId, Session]())
|
||||
|
||||
proc create*(server: IpldServer; id: SessionId; label, args: string) =
|
||||
if not server.sessions.contains id:
|
||||
try:
|
||||
let
|
||||
session = newSession(server.env, server.store, id, label, args)
|
||||
cap = server.env.ep.manage(session)
|
||||
server.sessions[id] = session
|
||||
proc deliverSession(env: GenodeEnv; id: SessionId; cap: IpldSessionCapability) {.
|
||||
importcpp: "#->parent().deliver_session_cap(Genode::Parent::Server::Id{#}, #)".}
|
||||
server.env.deliverSession(id, cap)
|
||||
echo "session opened for ", label
|
||||
except:
|
||||
echo "failed to create session for '", label, "', ", getCurrentExceptionMsg()
|
||||
server.env.sessionResponseDeny id
|
||||
|
||||
proc close*(server: IpldServer; id: SessionId) =
|
||||
## Close a session at the IPLD server.
|
||||
if server.sessions.contains id:
|
||||
let session = server.sessions[id]
|
||||
server.env.ep.dissolve(session)
|
||||
server.sessions.del id
|
||||
server.env.sessionResponseClose id
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
#
|
||||
# \brief IPLD session definitions
|
||||
# \author Emery Hemingway
|
||||
# \date 2017-11-11
|
||||
#
|
||||
|
||||
#
|
||||
# Copyright (C) 2017 Genode Labs GmbH
|
||||
#
|
||||
# This file is part of the Genode OS framework, which is distributed
|
||||
# under the terms of the GNU Affero General Public License version 3.
|
||||
#
|
||||
|
||||
import ipld
|
||||
|
||||
const MaxPacketSize* = 1 shl 18;
|
||||
|
||||
type
|
||||
IpldSessionCapability* {.final, pure,
|
||||
importcpp: "Ipld::Session_capability",
|
||||
header: "<ipld_session/capability.h>".} = object
|
||||
|
||||
IpldPacket* {.
|
||||
importcpp: "Ipld::Packet",
|
||||
header: "<ipld_session/ipld_session.h>".} = object
|
||||
|
||||
IpldOpcode* {.importcpp: "Ipld::Packet::Opcode".} = enum
|
||||
PUT, GET, INVALID
|
||||
|
||||
IpldError* {.importcpp: "Ipld::Packet::Error".} = enum
|
||||
OK, MISSING, OVERSIZE, FULL, ERROR
|
||||
|
||||
proc size*(pkt: IpldPacket): csize {.importcpp.}
|
||||
## Physical packet size.
|
||||
|
||||
proc cidStr(p: IpldPacket): cstring {.importcpp: "#.cid().string()".}
|
||||
proc cid*(p: IpldPacket): Cid = parseCid $p.cidStr
|
||||
proc setCid*(p: var IpldPacket; cid: cstring) {.importcpp: "#.cid(@)".}
|
||||
proc setCid*(p: var IpldPacket; cid: Cid) = p.setCid(cid.toHex())
|
||||
|
||||
proc operation*(pkt: IpldPacket): IpldOpcode {.importcpp.}
|
||||
proc len*(pkt: IpldPacket): csize {.importcpp: "length".}
|
||||
## Logical packet length.
|
||||
proc setLen*(pkt: var IpldPacket; len: int) {.importcpp: "length".}
|
||||
## Set logical packet length.
|
||||
proc error*(pkt: IpldPacket): IpldError {.importcpp.}
|
||||
proc setError*(pkt: var IpldPacket; err: IpldError) {.importcpp: "error".}
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
#
|
||||
# \brief IPLD session definitions
|
||||
# \author Emery Hemingway
|
||||
# \date 2017-11-11
|
||||
#
|
||||
|
||||
#
|
||||
# Copyright (C) 2017 Genode Labs GmbH
|
||||
#
|
||||
# This file is part of the Genode OS framework, which is distributed
|
||||
# under the terms of the GNU Affero General Public License version 3.
|
||||
#
|
||||
|
||||
import ipld
|
||||
|
||||
const MaxPacketSize* = 1 shl 18;
|
||||
|
||||
type
|
||||
IpldSessionCapability* {.final, pure,
|
||||
importcpp: "Ipld::Session_capability",
|
||||
header: "<ipld_session/capability.h>".} = object
|
||||
|
||||
IpldPacket* {.
|
||||
importcpp: "Ipld::Packet",
|
||||
header: "<ipld_session/ipld_session.h>".} = object
|
||||
|
||||
IpldOpcode* {.importcpp: "Ipld::Packet::Opcode".} = enum
|
||||
PUT, GET, INVALID
|
||||
|
||||
IpldError* {.importcpp: "Ipld::Packet::Error".} = enum
|
||||
OK, MISSING, OVERSIZE, FULL, ERROR
|
||||
|
||||
proc size*(pkt: IpldPacket): csize {.importcpp.}
|
||||
## Physical packet size.
|
||||
|
||||
proc cidStr(p: IpldPacket): cstring {.importcpp: "#.cid().string()".}
|
||||
proc cid*(p: IpldPacket): Cid = parseCid $p.cidStr
|
||||
proc setCid*(p: var IpldPacket; cid: cstring) {.importcpp: "#.cid(@)".}
|
||||
proc setCid*(p: var IpldPacket; cid: Cid) = p.setCid(cid.toHex())
|
||||
|
||||
proc operation*(pkt: IpldPacket): IpldOpcode {.importcpp.}
|
||||
proc len*(pkt: IpldPacket): csize {.importcpp: "length".}
|
||||
## Logical packet length.
|
||||
proc setLen*(pkt: var IpldPacket; len: int) {.importcpp: "length".}
|
||||
## Set logical packet length.
|
||||
proc error*(pkt: IpldPacket): IpldError {.importcpp.}
|
||||
proc setError*(pkt: var IpldPacket; err: IpldError) {.importcpp: "error".}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
import httpclient, json, base58.bitcoin, streams, nimSHA2, cbor, tables
|
||||
import httpclient, json, base58/bitcoin, streams, nimSHA2, cbor, tables
|
||||
|
||||
import ipld, multiformats, ipldstore, unixfs
|
||||
import ../ipld, ./multiformats, ./stores, ./unixfs
|
||||
|
||||
type
|
||||
IpfsStore* = ref IpfsStoreObj
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
import streams, os, parseopt
|
||||
|
||||
import ipfsdaemon, ipldstore, ipld, unixfs
|
||||
import ../ipld, ./ipfsdaemon, ./stores, ./unixfs
|
||||
|
||||
proc readFile(store: IpldStore; s: Stream; cid: Cid) =
|
||||
var chunk = ""
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
import nre, os, strutils, tables, parseopt, streams, cbor
|
||||
|
||||
import ipld, ipldstore, unixfs, multiformats
|
||||
import ../ipld, ./stores, ./unixfs, ./multiformats
|
||||
|
||||
type
|
||||
EvalError = object of SystemError
|
||||
|
|
@ -232,7 +232,7 @@ proc print(ast: NodeRef; s: Stream) =
|
|||
|
||||
proc readAtom(r: Reader): Atom =
|
||||
let token = r.next
|
||||
try:
|
||||
block:
|
||||
if token[token.low] == '"':
|
||||
if token[token.high] != '"':
|
||||
newAtomError("invalid string '$1'" % token)
|
||||
|
|
@ -241,12 +241,12 @@ proc readAtom(r: Reader): Atom =
|
|||
elif token.contains DirSep:
|
||||
# TODO: memoize this, store a table of paths to atoms
|
||||
newAtomPath token
|
||||
elif token.len > 48:
|
||||
elif token.len == 46 or token.len > 48:
|
||||
Atom(kind: atomCid, cid: token.parseCid)
|
||||
else:
|
||||
Atom(kind: atomSymbol, sym: token.normalize)
|
||||
except:
|
||||
newAtomError(getCurrentExceptionMsg())
|
||||
#except:
|
||||
# newAtomError(getCurrentExceptionMsg())
|
||||
|
||||
proc readForm(r: Reader): NodeRef
|
||||
|
||||
|
|
@ -399,7 +399,7 @@ proc lsFunc(env: Env; args: NodeObj): NodeRef =
|
|||
result = newNodeList()
|
||||
for n in args.walk:
|
||||
let a = n.atom
|
||||
if a.cid.isDagCbor:
|
||||
if a.cid.isDag:
|
||||
let ufsNode = env.getUnixfs a.cid
|
||||
if ufsNode.isDir:
|
||||
for name, u in ufsNode.items:
|
||||
|
|
@ -410,7 +410,7 @@ proc lsFunc(env: Env; args: NodeObj): NodeRef =
|
|||
e.append name.newAtomString.newNode
|
||||
result.append e
|
||||
else:
|
||||
raiseAssert("ls over a raw IPLD block")
|
||||
raiseAssert("ls over " & $a.cid.codec)
|
||||
|
||||
proc mapFunc(env: Env; args: NodeObj): NodeRef =
|
||||
assertArgCount(args, 2)
|
||||
|
|
@ -0,0 +1 @@
|
|||
-d:ssl
|
||||
|
|
@ -1,5 +1,4 @@
|
|||
import streams, strutils, os, ipld, cbor, multiformats, hex,
|
||||
ipldstore, ipldclient
|
||||
import streams, strutils, os, ../ipld, cbor, ./multiformats, ./store
|
||||
|
||||
type
|
||||
IpldReplicator* = ref IpldReplicatorObj
|
||||
|
|
@ -1,4 +1,6 @@
|
|||
import streams, strutils, os, ipld, cbor, multiformats, hex
|
||||
import std/streams, std/strutils, std/os
|
||||
import cbor
|
||||
import ../ipld, ./multiformats, ./priv/hex
|
||||
|
||||
type
|
||||
MissingObject* = ref object of SystemError
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
import strutils, multiformats, streams, tables, cbor, os, hex, math
|
||||
import strutils, multiformats, streams, tables, cbor, os, math
|
||||
|
||||
import ipld, ipldstore
|
||||
import ../ipld, ./stores
|
||||
|
||||
type EntryKey = enum
|
||||
typeKey = 1,
|
||||
Loading…
Reference in New Issue