From 2a8ef1737a3317cf53d24e0120d614bd90aaf7a4 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Sun, 19 May 2019 09:48:00 +0200 Subject: [PATCH] Blobsets slides --- build-x86_64/tup.config | 4 +- nim/musicbrainz_playlist_generator | 2 +- runtimes/audio_player/Tupfile | 9 ++ runtimes/audio_player/runtime | 27 ++++ talk/blobsets/slides.tex | 235 +++++++++++++++++++++++++++++ 5 files changed, 275 insertions(+), 2 deletions(-) create mode 100644 runtimes/audio_player/Tupfile create mode 100644 runtimes/audio_player/runtime create mode 100644 talk/blobsets/slides.tex diff --git a/build-x86_64/tup.config b/build-x86_64/tup.config index 1a1a03c90..8bbfe2275 100644 --- a/build-x86_64/tup.config +++ b/build-x86_64/tup.config @@ -14,8 +14,10 @@ CONFIG_UPSTREAM_VERSION=19.02 CONFIG_PUBLIC_USER=ehmry CONFIG_PUBLIC_KEY=6E10414EE7160781F5FEE8FFB14D043A26B3D2C7 +CONFIG_PUBLIC_PKG_AUDIO_PLAYER=ehmry/pkg/audio_player/2019-05-15 CONFIG_PUBLIC_PKG_AVPLAY=ehmry/pkg/avplay/2019-04-24 -CONFIG_PUBLIC_PKG_PDF_VIEW=nfeske/pkg/pdf_view/2019-03-21 +CONFIG_PUBLIC_PKG_PDF_VIEW=ehmry/pkg/pdf_view/2019-05-17 + CONFIG_PUBLIC_SRC_BASE_NOVA=genodelabs/src/base-nova/2019-03-17 CONFIG_PUBLIC_SRC_BLOCK_SHRED=ehmry/src/block_shred/2019-03-31 CONFIG_PUBLIC_SRC_CHROOT=genodelabs/src/chroot/2019-03-17 diff --git a/nim/musicbrainz_playlist_generator b/nim/musicbrainz_playlist_generator index 0809882ba..38462acef 160000 --- a/nim/musicbrainz_playlist_generator +++ b/nim/musicbrainz_playlist_generator @@ -1 +1 @@ -Subproject commit 0809882ba78cb0eaf9bd0c67edf911ef3385489b +Subproject commit 38462acefbb03254adca34387f763de1019e4aa8 diff --git a/runtimes/audio_player/Tupfile b/runtimes/audio_player/Tupfile new file mode 100644 index 000000000..fb58cf0c3 --- /dev/null +++ b/runtimes/audio_player/Tupfile @@ -0,0 +1,9 @@ +TARGET_NAME = audio_player + +PKG_DEPENDS += \ + @(PUBLIC_PKG_AUDIO_PLAYER) \ + +include_rules + +: runtime |> !collect_pkg_runtime |> +include $(PKG_RULES) diff --git a/runtimes/audio_player/runtime b/runtimes/audio_player/runtime new file mode 100644 index 000000000..4c940d003 --- /dev/null +++ b/runtimes/audio_player/runtime @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/talk/blobsets/slides.tex b/talk/blobsets/slides.tex new file mode 100644 index 000000000..3abc156e8 --- /dev/null +++ b/talk/blobsets/slides.tex @@ -0,0 +1,235 @@ +\documentclass{beamer} +\usepackage{listings} + +\usepackage{tikz} +\usetikzlibrary{shapes,arrows,positioning} + +\usetheme{Dresden} +\title[Blobsets]{Blobsets} +\author{Emery} + +\begin{document} + +\begin{frame} +\titlepage +\end{frame} + +\begin{frame}{Motivation} +Hierarchical storage is a poor abstraction for data with rich context +\pause +\\Hierarchy is weak metadata +\pause +\\Data at rest accumulates contextual metadata +\end{frame} + +\begin{frame}{UNIX / Plan 9 file-system} +\begin{itemize} +\pause \item Ergonomic, intuitive +\pause \item Stable semantics +\pause \item Optimized +\pause \item Multipurpose - bulk storage, configuration, system state, UEFI variables +\pause \item Archaic sharing model +\pause \item Migration can be messy +\pause \item SPOF +\end{itemize} +\end{frame} + +\begin{frame}{IPFS} +\begin{itemize} +\pause \item Content addressed +\pause \item Explicit namespacing +\pause \item Distributed +\pause \item Global network state - DHT +\pause \item LevelDB +\pause \item Super daemon +\pause \item Extensible +\end{itemize} +\end{frame} + +\begin{frame}{DAT} +\begin{itemize} +\pause \item Javascript +\end{itemize} +\end{frame} + +\begin{frame}{An alternative approach} +Metadata is too rich for the storage layer\\ +\begin{itemize} +\pause \item Storage without implicit structure +\pause \item Fixed, non-negotiable specification +\pause \item Deterministic, transactional updates +\pause \item Multiple least-privileged components +\pause \item Replication +\pause \item Network optional +\end{itemize} +\end{frame} + +\section{Blobsets} + +\begin{frame}{Just a hash map} +\pause +Blobs may be data of any lengths identified by a 256 bit hash. +\\~\ +\pause + +Sets are collections of blobs indexed by 64 bit keys. +\\~\ +\pause + +Keys may be derived from a hash function. +\\~\ +\pause + +Set may contain the same blob under different keys. +\\~\ +\pause + +Sets are not recursive. +\end{frame} + +\begin{frame}{Implementation} + +A set is a hash array mapped trie +\\~\ +\pause + +Set serialization is CBOR +\\~\ +\pause + +Blob hash function is a BLAKE2B-256 tree hash with 64 KiB leaves +\\~\ +\pause + +Key hash function is SipHash +\\~\ +\pause + +Approximately 1 KLOC of Nim +\\~\ + +\end{frame}{} + +\begin{frame}{BlobStore API} +\pause +\begin{block}{\lstinline{openBlobStream(BlobStore, BlobHash) -> BlobStream}} + \lstinline{get_pos(BlobStream) -> Integer} + + \lstinline{set_pos(BlobStream, Integer)} + + \lstinline{read(BlobStream, Integer) -> String} + + \lstinline{close(BlobStream)} +\end{block} +\begin{block}{\lstinline{openIngestStream(BlobStore store) -> IngestStream}} + \lstinline{write(IngestStream, String)} + + \lstinline{finish(IngestStream) -> BlobHash} + + \lstinline{cancel(IngestStream)} + +\end{block} + +\begin{block}{\lstinline{close(BlobStore store)}} +\end{block} + +\end{frame} + +\begin{frame}{BlobSet API} +\lstinline{load(BlobStore, BlobHash) -> BlobSet} +\\~\ + +\lstinline{insert(BlobStore, BlobSet, Key, BlobHash) -> BlobSet} +\\~\ + +\lstinline{remove(BlobStore, BlobSet, Key) -> BlobSet} +\\~\ + +\lstinline{union(BlobStore, BlobSet, BlobSet) -> BlobSet} +\\~\ + +\lstinline{apply(BlobStore, BlobSet, Key, Procedure)} +\\~\ + +\lstinline{randomApply(BlobStore, BlobSet, Procedure)} +\\~\ + +\lstinline{commit(BlobStore, BlobSet) -> BlobSet} +\end{frame} + +\section{Demo} +\begin{frame}{} +\center{REPL DEMO} +\end{frame} + +\section{Audio player} +\begin{frame}{Music Problem} +\begin{itemize} +\pause \item New operating system architecture +\pause \item Restricted access to audio hardware +\pause \item Audio player must be isolated +\pause \item Native audio player has no GUI or CLI +\pause \item Context, not files +\end{itemize} +\end{frame} + +\begin{frame}{} +\tikzstyle{block} = [draw, rectangle, minimum height=3em, minimum width=3em] +\tikzstyle{virtual} = [coordinate] +\begin{tikzpicture}[>=stealth,auto, node distance=1cm] + % Place nodes + \node [block] (playlist) {XSPF playlist}; + \node [block, left=of playlist, xshift=-8ex] (ext2) {EXT2}; + \node [block, above=of playlist] (audioplayer) {audio player}; + \node [block, above=of ext2] (blob_fs) {synthetic blob file-system}; + \node [block, below=of playlist] (vm) {VM}; + \node [block, below=of ext2] (blob_http) {blob HTTP service}; + + % Connect nodes + \draw [->] (blob_fs) -- node {} (ext2); + \draw [->] (blob_http) -- node {} (ext2); + \draw [->] (audioplayer) -- node {} (blob_fs); + \draw [->] (vm) -- node {} (blob_http); + \draw [->] (audioplayer) -- node {} (playlist); + \draw [->] (vm) -- node {} (playlist); +\end{tikzpicture} +\end{frame} + +\begin{frame}{MusicBrainz} + \includegraphics[height=.8\textheight,keepaspectratio]{ngs.png} +\end{frame} + +\begin{frame}{MusicBrainz} + +\begin{block} +{\url{https://musicbrainz.org/recording/7053937d-45b0-4303-ace7-0db8586821bf}} +\end{block} + +\begin{block} +{\url{https://musicbrainz.org/recording/7a677b2a-0a3b-4f62-a1fb-1385ed5c743a}} +\end{block} + +\begin{block} +{\url{https://musicbrainz.org/recording/20ff3e70-f7af-4417-b187-0935c526aebc}} +\end{block} + +\begin{block} +{\url{http://musicbrainz.org/recording/7dd9713e-0515-4011-937d-969c3a445182}} +\end{block} + +\begin{block} +{\url{https://musicbrainz.org/release/76f203a1-e7f4-45a6-b8af-0bea2bdc1c5c}} +\end{block} + +\end{frame} + +\begin{frame}{Further work} +\begin{itemize} +\pause \item Replication layer +\pause \item Lazy loading +\pause \item Garbage collection +\pause \item Books, movies, software +\end{itemize} +\end{frame} + +\end{document}