ehmry
/
genode-ehmry
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Blobsets slides

remotes/origin/19.02
Emery Hemingway 2019-05-19 09:48:00 +02:00
parent 18a47218ec
commit 2a8ef1737a
5 changed files with 275 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
build-x86_64/tup.config
View File

@ -14,8 +14,10 @@ CONFIG_UPSTREAM_VERSION=19.02
CONFIG_PUBLIC_USER=ehmry
CONFIG_PUBLIC_KEY=6E10414EE7160781F5FEE8FFB14D043A26B3D2C7
CONFIG_PUBLIC_PKG_AUDIO_PLAYER=ehmry/pkg/audio_player/2019-05-15
CONFIG_PUBLIC_PKG_AVPLAY=ehmry/pkg/avplay/2019-04-24
CONFIG_PUBLIC_PKG_PDF_VIEW=nfeske/pkg/pdf_view/2019-03-21
CONFIG_PUBLIC_PKG_PDF_VIEW=ehmry/pkg/pdf_view/2019-05-17
CONFIG_PUBLIC_SRC_BASE_NOVA=genodelabs/src/base-nova/2019-03-17
CONFIG_PUBLIC_SRC_BLOCK_SHRED=ehmry/src/block_shred/2019-03-31
CONFIG_PUBLIC_SRC_CHROOT=genodelabs/src/chroot/2019-03-17

2
nim/musicbrainz_playlist_generator

@ -1 +1 @@
Subproject commit 0809882ba78cb0eaf9bd0c67edf911ef3385489b
Subproject commit 38462acefbb03254adca34387f763de1019e4aa8

9
runtimes/audio_player/Tupfile Normal file
View File

@ -0,0 +1,9 @@
TARGET_NAME = audio_player
PKG_DEPENDS += \
@(PUBLIC_PKG_AUDIO_PLAYER) \
include_rules
: runtime |> !collect_pkg_runtime |>
include $(PKG_RULES)

27
runtimes/audio_player/runtime Normal file
View File

@ -0,0 +1,27 @@
<runtime ram="32M" caps="200" binary="audio_player">
<requires>
<audio_out/>
<file_system/>
<rom label="playlist"/>
</requires>
<content>
<rom label="ld.lib.so"/>
<rom label="audio_player"/>
<rom label="libc.lib.so"/>
<rom label="libm.lib.so"/>
<rom label="zlib.lib.so"/>
<rom label="vfs.lib.so"/>
<rom label="avcodec.lib.so"/>
<rom label="avfilter.lib.so"/>
<rom label="avformat.lib.so"/>
<rom label="avresample.lib.so"/>
<rom label="avutil.lib.so"/>
</content>
<config state="playing" scan_playlist="yes">
<vfs> <fs label="c4e7043e7b234309115c5e87b358b85623c84a35a39deff87a044d88814bd862" writeable="no"/> </vfs>
</config>
</runtime>

235
talk/blobsets/slides.tex Normal file
View File

@ -0,0 +1,235 @@
\documentclass{beamer}
\usepackage{listings}
\usepackage{tikz}
\usetikzlibrary{shapes,arrows,positioning}
\usetheme{Dresden}
\title[Blobsets]{Blobsets}
\author{Emery}
\begin{document}
\begin{frame}
\titlepage
\end{frame}
\begin{frame}{Motivation}
Hierarchical storage is a poor abstraction for data with rich context
\pause
\\Hierarchy is weak metadata
\pause
\\Data at rest accumulates contextual metadata
\end{frame}
\begin{frame}{UNIX / Plan 9 file-system}
\begin{itemize}
\pause \item Ergonomic, intuitive
\pause \item Stable semantics
\pause \item Optimized
\pause \item Multipurpose - bulk storage, configuration, system state, UEFI variables
\pause \item Archaic sharing model
\pause \item Migration can be messy
\pause \item SPOF
\end{itemize}
\end{frame}
\begin{frame}{IPFS}
\begin{itemize}
\pause \item Content addressed
\pause \item Explicit namespacing
\pause \item Distributed
\pause \item Global network state - DHT
\pause \item LevelDB
\pause \item Super daemon
\pause \item Extensible
\end{itemize}
\end{frame}
\begin{frame}{DAT}
\begin{itemize}
\pause \item Javascript
\end{itemize}
\end{frame}
\begin{frame}{An alternative approach}
Metadata is too rich for the storage layer\\
\begin{itemize}
\pause \item Storage without implicit structure
\pause \item Fixed, non-negotiable specification
\pause \item Deterministic, transactional updates
\pause \item Multiple least-privileged components
\pause \item Replication
\pause \item Network optional
\end{itemize}
\end{frame}
\section{Blobsets}
\begin{frame}{Just a hash map}
\pause
Blobs may be data of any lengths identified by a 256 bit hash.
\\~\
\pause
Sets are collections of blobs indexed by 64 bit keys.
\\~\
\pause
Keys may be derived from a hash function.
\\~\
\pause
Set may contain the same blob under different keys.
\\~\
\pause
Sets are not recursive.
\end{frame}
\begin{frame}{Implementation}
A set is a hash array mapped trie
\\~\
\pause
Set serialization is CBOR
\\~\
\pause
Blob hash function is a BLAKE2B-256 tree hash with 64 KiB leaves
\\~\
\pause
Key hash function is SipHash
\\~\
\pause
Approximately 1 KLOC of Nim
\\~\
\end{frame}{}
\begin{frame}{BlobStore API}
\pause
\begin{block}{\lstinline{openBlobStream(BlobStore, BlobHash) -> BlobStream}}
\lstinline{get_pos(BlobStream) -> Integer}
\lstinline{set_pos(BlobStream, Integer)}
\lstinline{read(BlobStream, Integer) -> String}
\lstinline{close(BlobStream)}
\end{block}
\begin{block}{\lstinline{openIngestStream(BlobStore store) -> IngestStream}}
\lstinline{write(IngestStream, String)}
\lstinline{finish(IngestStream) -> BlobHash}
\lstinline{cancel(IngestStream)}
\end{block}
\begin{block}{\lstinline{close(BlobStore store)}}
\end{block}
\end{frame}
\begin{frame}{BlobSet API}
\lstinline{load(BlobStore, BlobHash) -> BlobSet}
\\~\
\lstinline{insert(BlobStore, BlobSet, Key, BlobHash) -> BlobSet}
\\~\
\lstinline{remove(BlobStore, BlobSet, Key) -> BlobSet}
\\~\
\lstinline{union(BlobStore, BlobSet, BlobSet) -> BlobSet}
\\~\
\lstinline{apply(BlobStore, BlobSet, Key, Procedure)}
\\~\
\lstinline{randomApply(BlobStore, BlobSet, Procedure)}
\\~\
\lstinline{commit(BlobStore, BlobSet) -> BlobSet}
\end{frame}
\section{Demo}
\begin{frame}{}
\center{REPL DEMO}
\end{frame}
\section{Audio player}
\begin{frame}{Music Problem}
\begin{itemize}
\pause \item New operating system architecture
\pause \item Restricted access to audio hardware
\pause \item Audio player must be isolated
\pause \item Native audio player has no GUI or CLI
\pause \item Context, not files
\end{itemize}
\end{frame}
\begin{frame}{}
\tikzstyle{block} = [draw, rectangle, minimum height=3em, minimum width=3em]
\tikzstyle{virtual} = [coordinate]
\begin{tikzpicture}[>=stealth,auto, node distance=1cm]
% Place nodes
\node [block] (playlist) {XSPF playlist};
\node [block, left=of playlist, xshift=-8ex] (ext2) {EXT2};
\node [block, above=of playlist] (audioplayer) {audio player};
\node [block, above=of ext2] (blob_fs) {synthetic blob file-system};
\node [block, below=of playlist] (vm) {VM};
\node [block, below=of ext2] (blob_http) {blob HTTP service};
% Connect nodes
\draw [->] (blob_fs) -- node {} (ext2);
\draw [->] (blob_http) -- node {} (ext2);
\draw [->] (audioplayer) -- node {} (blob_fs);
\draw [->] (vm) -- node {} (blob_http);
\draw [->] (audioplayer) -- node {} (playlist);
\draw [->] (vm) -- node {} (playlist);
\end{tikzpicture}
\end{frame}
\begin{frame}{MusicBrainz}
\includegraphics[height=.8\textheight,keepaspectratio]{ngs.png}
\end{frame}
\begin{frame}{MusicBrainz}
\begin{block}
{\url{https://musicbrainz.org/recording/7053937d-45b0-4303-ace7-0db8586821bf}}
\end{block}
\begin{block}
{\url{https://musicbrainz.org/recording/7a677b2a-0a3b-4f62-a1fb-1385ed5c743a}}
\end{block}
\begin{block}
{\url{https://musicbrainz.org/recording/20ff3e70-f7af-4417-b187-0935c526aebc}}
\end{block}
\begin{block}
{\url{http://musicbrainz.org/recording/7dd9713e-0515-4011-937d-969c3a445182}}
\end{block}
\begin{block}
{\url{https://musicbrainz.org/release/76f203a1-e7f4-45a6-b8af-0bea2bdc1c5c}}
\end{block}
\end{frame}
\begin{frame}{Further work}
\begin{itemize}
\pause \item Replication layer
\pause \item Lazy loading
\pause \item Garbage collection
\pause \item Books, movies, software
\end{itemize}
\end{frame}
\end{document}