aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAlex Auvolat <alex@adnab.me>2018-11-06 15:03:22 +0100
committerAlex Auvolat <alex@adnab.me>2018-11-06 15:03:22 +0100
commit2973cf99c5b677c71717d916f83212bc2e6b36dc (patch)
tree8651b326786c62fc4e0bc438a625a13bf5df1844
parentc4f6cbab20b0b1d08755073d93365e5bd00dc755 (diff)
downloadshard-2973cf99c5b677c71717d916f83212bc2e6b36dc.tar.gz
shard-2973cf99c5b677c71717d916f83212bc2e6b36dc.zip
Document
-rw-r--r--README.md5
-rw-r--r--shard/lib/app/chat.ex47
-rw-r--r--shard/lib/app/directory.ex21
-rw-r--r--shard/lib/app/file.ex23
-rw-r--r--shard/lib/app/identity.ex8
-rw-r--r--shard/lib/app/pagestore.ex52
-rw-r--r--shard/lib/cli/cli.ex21
-rw-r--r--shard/lib/data/data.ex22
-rw-r--r--shard/lib/data/merklelist.ex8
-rw-r--r--shard/lib/data/merklesearchtree.ex16
-rw-r--r--shard/lib/data/merkletree.ex14
-rw-r--r--shard/lib/data/signrev.ex2
-rw-r--r--shard/lib/data/store.ex11
-rw-r--r--shard/lib/keys.ex30
-rw-r--r--shard/lib/manager.ex44
-rw-r--r--shard/lib/net/addr.ex18
-rw-r--r--shard/lib/net/auth.ex10
-rw-r--r--shard/lib/net/group.ex12
-rw-r--r--shard/lib/net/manager.ex12
-rw-r--r--shard/lib/net/tcpconn.ex15
-rw-r--r--shard/lib/net/tcpserver.ex4
-rw-r--r--shard/lib/shard_uri.ex9
22 files changed, 295 insertions, 109 deletions
diff --git a/README.md b/README.md
index c247e4e..db52b04 100644
--- a/README.md
+++ b/README.md
@@ -86,6 +86,8 @@ Current status
What is available
-----------------
+All of these are rudimentary prototypes at an early stage.
+
* Chat rooms (public and private) with full history and efficient data structure for
retrieving missing messages after disconnection
* File upload (public only)
@@ -99,6 +101,8 @@ See `TODO` file for more details.
* Finding peers via DHT (very easy to add)
* Invite/notification system
* Good access control
+* Good networking behind NAT/Firewall
+* Automatic discovery on local networks
* More applications
How to use it?
@@ -217,4 +221,5 @@ This CLI supports a few basic commands:
- `/pm nickname1 [nickname2] [...]`: enter private conversation with someone
- `/send_file path`: make file available on the network and send link to current chat room.
**WARNING: all files are publicly available for now, even if they are sent in a private chat room.**
+- `/shards`: return the list of all shards on the system
- `/quit`: return to iex prompt
diff --git a/shard/lib/app/chat.ex b/shard/lib/app/chat.ex
index ff0c97d..b046fc3 100644
--- a/shard/lib/app/chat.ex
+++ b/shard/lib/app/chat.ex
@@ -12,11 +12,7 @@ defmodule SApp.Chat do
%SApp.Chat.PrivChat.Manifest{pk_list: ordered_list_of_authorized_pks}
Future improvements:
- - message signing
- - storage of the chatroom messages to disk
- use a DHT to find peers that are interested in this channel
- - epidemic broadcast (carefull not to be too costly,
- maybe by limiting the number of peers we talk to)
- partial synchronization only == data distributed over peers
"""
@@ -32,7 +28,9 @@ defmodule SApp.Chat do
defmodule Manifest do
@moduledoc"""
- Manifest for a public chat room defined by its name.
+ Manifest for a public chat room defined by its name. Example:
+
+ %SApp.Chat.Manifest{channel: "test"}
"""
defstruct [:channel]
@@ -73,19 +71,22 @@ defmodule SApp.Chat do
# ==========
defmodule State do
+ @moduledoc"""
+ Internal state struct of chat shard.
+ """
+
defstruct [:id, :netgroup, :manifest, :page_store, :mst, :subs, :read]
end
@doc """
- Start a process that connects to a given channel
+ Start a process that connects to a given channel. Don't call directly, use for instance:
+
+ Shard.Manager.find_or_start %SApp.Chat.Manifest{channel: "my_chan"}
"""
def start_link(manifest) do
GenServer.start_link(__MODULE__, manifest)
end
- @doc """
- Initialize channel process.
- """
def init(manifest) do
id = SData.term_hash manifest
@@ -103,7 +104,7 @@ defmodule SApp.Chat do
end
root = cond do
root == nil -> nil
- GenServer.call(page_store, {:have_rec, root}) -> root
+ SApp.PageStore.have_rec?(page_store, root) -> root
true ->
Logger.warn "Not all pages for saved root were saved, restarting from an empty state!"
nil
@@ -124,9 +125,6 @@ defmodule SApp.Chat do
}
end
- @doc """
- Implementation of the :manifest call that returns the chat room's manifest
- """
def handle_call(:manifest, _from, state) do
{:reply, state.manifest, state}
end
@@ -166,11 +164,6 @@ defmodule SApp.Chat do
{:noreply, state}
end
- @doc """
- Implementation of the :chat_send handler. This is the main handler that is used
- to send a message to the chat room. Puts the message in the store and syncs
- with all connected peers.
- """
def handle_cast({:chat_send, pk, msg}, state) do
next_ts = case MST.last(state.mst, nil, 1) do
[] -> System.os_time :seconds
@@ -204,10 +197,6 @@ defmodule SApp.Chat do
{:noreply, state}
end
- @doc """
- Implementation of the :interested handler, this is called when a peer we are
- connected to asks to recieve data for this channel.
- """
def handle_cast({:interested, conn_pid, auth}, state) do
if SNet.Group.in_group?(state.netgroup, conn_pid, auth) do
SNet.Manager.send_pid(conn_pid, {state.id, nil, {:root, state.mst.root, true}})
@@ -221,10 +210,6 @@ defmodule SApp.Chat do
{:noreply, %{ state | subs: new_subs }}
end
- @doc """
- Implementation of the :msg handler, which is the main handler for messages
- comming from other peers concerning this chat room.
- """
def handle_cast({:msg, conn_pid, auth, _shard_id, nil, msg}, state) do
if not SNet.Group.in_group?(state.netgroup, conn_pid, auth) do
# Ignore message
@@ -245,7 +230,7 @@ defmodule SApp.Chat do
mst2 = MST.insert(state.mst, msgitem)
if mst2.root == new_root do
state = %{state | mst: mst2}
- GenServer.cast(state.page_store, {:set_roots, [mst2.root]})
+ SApp.PageStore.set_roots(state.page_store, [mst2.root])
save_state(state)
msg_callback(state, msgitem)
SNet.Group.broadcast(state.netgroup, {state.id, nil, msg}, exclude_pid: [conn_pid])
@@ -311,7 +296,7 @@ defmodule SApp.Chat do
for x <- new do
msg_callback(state, x)
end
- GenServer.cast(state.page_store, {:set_roots, [mst.root]})
+ SApp.PageStore.set_roots(state.page_store, [mst.root])
state = %{state | mst: mst}
save_state(state)
if state.mst.root != old_root do
@@ -365,15 +350,15 @@ defmodule SApp.Chat do
The process calling this function will start recieving messages of the form:
- {:chat_recv, manifest, {pk, msgbin, sign}}
+ {:chat_recv, manifest, {pk, msgbin, sign}}
or
- {:chat_send, manifest, {pk, msgbin, sign}}
+ {:chat_send, manifest, {pk, msgbin, sign}}
msgbin can be used in the following way:
- {timestamp, message} = SData.term_unbin msgbin
+ {timestamp, message} = SData.term_unbin msgbin
"""
def subscribe(shard_pid) do
GenServer.cast(shard_pid, {:subscribe, self()})
diff --git a/shard/lib/app/directory.ex b/shard/lib/app/directory.ex
index cbea8c3..257e8b9 100644
--- a/shard/lib/app/directory.ex
+++ b/shard/lib/app/directory.ex
@@ -29,9 +29,18 @@ defmodule SApp.Directory do
end
defmodule State do
+ @moduledoc"""
+ Internal state struct of directory shard.
+ """
+
defstruct [:owner, :public, :name, :manifest, :id, :netgroup, :items, :revitems]
end
+ @doc"""
+ Start a process that connects to a given channel. Don't call directly, use for instance:
+
+ Shard.Manager.find_or_start %SApp.Directory.Manifest{owner: my_pk, public: false, name: "collection"}
+ """
def start_link(manifest) do
GenServer.start_link(__MODULE__, manifest)
end
@@ -217,23 +226,23 @@ defmodule SApp.Directory do
@doc"""
Return list of items stored in this directory.
- Returns a dictionnary of %{name => {manifest, stored?}}.
+ Returns a dictionnary of `%{name => {manifest, stored?}}`.
"""
def get_items(pid) do
GenServer.call(pid, :get_items)
end
@doc"""
- Return the manifest of item with a given name in directory, or nil if not found.
+ Return the manifest of item with a given name in directory, or `nil` if not found.
- Equivalent to get_items(pid)[name] but better.
+ Equivalent to `get_items(pid)[name]` but better.
"""
def read(pid, name) do
GenServer.call(pid, {:read, name})
end
@doc"""
- Find an item in the directory by its manifest. Returns name if found or nil if not found.
+ Find an item in the directory by its manifest. Returns name if found or `nil` if not found.
"""
def find(pid, manifest) do
GenServer.call(pid, {:find, manifest})
@@ -241,8 +250,8 @@ defmodule SApp.Directory do
@doc"""
Add an item to this directory. An item is a name for a shard manifest.
- An item added to a directory becomes a dependency of the directory, i.e.
- if the directory is pinned then all items inside are pinned as well.
+ An item added to a directory with `stored = true` becomes a dependency of the directory,
+ i.e. if the directory is pinned then all items inside are pinned as well.
"""
def add_item(pid, name, manifest, stored \\ true) do
GenServer.call(pid, {:add_item, name, manifest, stored})
diff --git a/shard/lib/app/file.ex b/shard/lib/app/file.ex
index e2a9798..0e07cc3 100644
--- a/shard/lib/app/file.ex
+++ b/shard/lib/app/file.ex
@@ -9,9 +9,12 @@ defmodule SApp.File do
file_hash: hash
size: int
mime_type: string
- }
+ }
+
+ The file is cut in blocks that are collected in a k-ary Merkle tree
+ (see SData.MerkleTree for block size and k value).
- The file is cut in blocks of 4kb that are collected in a 64-ary Merkle tree.
+ TODO I feel bad about some parts of the logic in here.
"""
use GenServer
@@ -26,7 +29,7 @@ defmodule SApp.File do
defmodule Manifest do
@moduledoc"""
Manifest for a file.
- The file is identified by the root hash of its Merkle tree and by its mime type.
+ The file is identified by its infohash, which is the hash of a `SApp.File.Info` struct.
"""
defstruct [:infohash]
@@ -46,9 +49,21 @@ defmodule SApp.File do
end
defmodule State do
+ @moduledoc"""
+ Internal state struct for file shard.
+ """
defstruct [:infohash, :id, :manifest, :netgroup, :info, :infobin, :store, :missing, :path, :reqs]
end
+ @doc """
+ Start a process that connects to a given channel. Don't call directly, use for instance:
+
+ Shard.Manager.find_or_start %SApp.File.Manifest{infohash: "some_infohash"}
+
+ or:
+
+ SApp.File.Create("/path/to/file", "mime/type")
+ """
def start_link(manifest) do
GenServer.start_link(__MODULE__, manifest)
end
@@ -229,7 +244,7 @@ defmodule SApp.File do
true ->
meta = get_mt(state)
n_blocks = MT.block_count(meta)
- expected_hashes = MT.get_range(meta, 0..(n_blocks-1))
+ expected_hashes = MT.get_all(meta)
actual_hashes = if File.exists?(state.path) do
File.stream!(state.path, [], MT.block_size())
|> Enum.map(&(:crypto.hash(:sha256, &1)))
diff --git a/shard/lib/app/identity.ex b/shard/lib/app/identity.ex
index 78abbe7..7422822 100644
--- a/shard/lib/app/identity.ex
+++ b/shard/lib/app/identity.ex
@@ -34,9 +34,17 @@ defmodule SApp.Identity do
end
defmodule State do
+ @moduledoc"""
+ Internal state struct for identity shard.
+ """
defstruct [:pk, :id, :state, :netgroup]
end
+ @doc """
+ Start a process that connects to a given channel. Don't call directly, use for instance:
+
+ Shard.Manager.find_or_start %SApp.Identity.Manifest{pk: some_public_key}
+ """
def start_link(manifest) do
GenServer.start_link(__MODULE__, manifest)
end
diff --git a/shard/lib/app/pagestore.ex b/shard/lib/app/pagestore.ex
index 3cda51d..0cbb10a 100644
--- a/shard/lib/app/pagestore.ex
+++ b/shard/lib/app/pagestore.ex
@@ -7,12 +7,29 @@ defmodule SApp.PageStore do
Uses an ETS table of:
- { page_id, why_have_it } -- waiting for data
- { page_id, why_have_it, data } -- once we have the data
+ { page_id, why_have_it } # waiting for data
+ { page_id, why_have_it, data } # once we have the data
- why_have_it := :root
- | {:req_by, some_other_page_id}
- | {:cached, expiry_date}
+ why_have_it := :root
+ | {:req_by, some_other_page_id}
+ | {:cached, expiry_date}
+
+ TODO: at the moment we are trying to pull all missing pages at once from our peers.
+ This can work for metadata that isn't too big but won't work with bigger objects.
+ Have a smart strategy where we limit the number of requests currently in-flight but
+ still make sure everything gets pulled in. This will also pave the way to selectively
+ pulling in pages, for instance if we have a function to give them a priority score and
+ a maximum stored page count.
+
+ A `SApp.PageStore` can be used as a `SData.PageStore` in the following way:
+
+ %SApp.PageStore{pid: store_pid}
+
+ or:
+
+ %SApp.PageStore{pid: store_pid, prefer_ask: [connection_pid, ...]}
+
+ In the second case, missing pages will be requested first to the specified peers.
"""
use GenServer
@@ -25,6 +42,9 @@ defmodule SApp.PageStore do
@max_failures 4 # Maximum of peers that reply not_found before we abandon
defmodule State do
+ @moduledoc"""
+ Internal state struct of pagestore process.
+ """
defstruct [:shard_id, :path, :netgroup, :store, :reqs, :retries, :store_path]
end
@@ -258,7 +278,7 @@ defmodule SApp.PageStore do
{:noreply, state}
end
- def ask_random_peers(state, key) do
+ defp ask_random_peers(state, key) do
SNet.Group.broadcast(state.netgroup, {state.shard_id, state.path, {:get, key}}, nmax: 3)
end
@@ -289,4 +309,24 @@ defmodule SApp.PageStore do
store ## DO SOMETHING???
end
end
+
+ # ====================
+ # PAGE STORE INTERFACE
+ # ====================
+
+ @doc"""
+ Returns `true` if the page store currently stores the specified root page
+ and all its dependencies, recursively.
+ """
+ def have_rec?(pid, root) do
+ GenServer.call(pid, {:have_rec, root})
+ end
+
+ @doc"""
+ Define the set of root pages we are interested in. This will start pulling in
+ the defined pages and all their dependencies recursively if we don't have them.
+ """
+ def set_roots(pid, roots) do
+ GenServer.cast(pid, {:set_roots, roots})
+ end
end
diff --git a/shard/lib/cli/cli.ex b/shard/lib/cli/cli.ex
index 54b882f..8495b93 100644
--- a/shard/lib/cli/cli.ex
+++ b/shard/lib/cli/cli.ex
@@ -1,12 +1,23 @@
defmodule SCLI do
@moduledoc """
- Small command line interface for the chat application
+ Small command line interface for the chat application. Supports public chat rooms,
+ private conversations, sending files (but not receiving them - could be done easily).
+
+ The code of this module is intended as an example of how to use the Shard library.
+
+ TODO: more commands.
"""
defmodule State do
+ @moduledoc"""
+ Internal state struct of the CLI.
+ """
defstruct [:room_pid, :id_pid, :pk]
end
+ @doc"""
+ Call this from the iex prompt to launch the CLI.
+ """
def run() do
for {_chid, manifest, _} <- Shard.Manager.list_shards do
case manifest do
@@ -197,6 +208,14 @@ defmodule SCLI do
state
end
+ defp handle_command(state, ["shards"]) do
+ Shard.Manager.list_shards
+ |> Enum.map(&(ShardURI.from_manifest(elem(&1, 1))))
+ |> Enum.sort()
+ |> Enum.map(&IO.puts/1)
+ state
+ end
+
defp handle_command(state, _cmd) do
IO.puts "Invalid command"
state
diff --git a/shard/lib/data/data.ex b/shard/lib/data/data.ex
index 33dca09..8d2b277 100644
--- a/shard/lib/data/data.ex
+++ b/shard/lib/data/data.ex
@@ -10,34 +10,46 @@ defmodule SData do
These functions must only return :duplicate for equal items.
"""
- @doc """
+ @doc"""
Calculate the hash of an Erlang term by first converting it to its
- binary representation.
+ binary representation. Equivalent to `bin_hash(term_bin(term))`.
"""
def term_hash(term, algo \\ :sha256) do
:crypto.hash(algo, (:erlang.term_to_binary term))
end
+ @doc"""
+ Convert any Erlang term to a binary representation.
+ """
def term_bin(term) do
:erlang.term_to_binary term
end
+ @doc"""
+ Calculate the hash of a binary.
+ """
def bin_hash(bin, algo \\ :sha256) do
:crypto.hash(algo, bin)
end
+ @doc"""
+ Calculate the hash of a file.
+ """
def file_hash(path, algo \\ :sha256) do
File.stream!(path, [], 65536)
|> Enum.reduce(:crypto.hash_init(algo), &(:crypto.hash_update(&2, &1)))
|> :crypto.hash_final()
end
+ @doc"""
+ Recover an Erlang term from its binary representation.
+ """
def term_unbin(bin) do
:erlang.binary_to_term(bin, [:safe])
end
@doc"""
- Compare function for arbitrary terms using the Erlang order
+ Compare function for arbitrary terms using the Erlang order
"""
def cmp_term(a, b) do
cond do
@@ -48,7 +60,7 @@ defmodule SData do
end
@doc"""
- Compare function for timestamped strings
+ Compare function for timestamped strings
"""
def cmp_ts_str({ts1, str1}, {ts2, str2}) do
cond do
@@ -61,7 +73,7 @@ defmodule SData do
end
@doc"""
- Merge function for nils
+ Merge function for nils
"""
def merge_true(true, true), do: true
end
diff --git a/shard/lib/data/merklelist.ex b/shard/lib/data/merklelist.ex
index 9b44ee8..c450ca7 100644
--- a/shard/lib/data/merklelist.ex
+++ b/shard/lib/data/merklelist.ex
@@ -1,12 +1,8 @@
defmodule SData.MerkleList do
@moduledoc"""
- A simple Merkle list store.
+ A simple Merkle list store. Not used.
- Future improvements:
- - When messages are inserted other than at the top, all intermediate hashes
- change. Keep track of the mapping from old hashes to new hashes so that get
- requests can work even for hashes that are not valid anymore.
- - group items in "pages" (bigger bundles)
+ TODO delete this module
"""
defstruct [:root, :top, :cmp, :store]
diff --git a/shard/lib/data/merklesearchtree.ex b/shard/lib/data/merklesearchtree.ex
index e646774..f67843d 100644
--- a/shard/lib/data/merklesearchtree.ex
+++ b/shard/lib/data/merklesearchtree.ex
@@ -3,15 +3,15 @@ defmodule SData.MerkleSearchTree do
A Merkle search tree.
A node of the tree is
- {
- level,
- hash_of_node | nil,
- [
- { item_low_bound, hash_of_node | nil },
- { item_low_bound, hash_of_node | nil },
- ...
+ {
+ level,
+ hash_of_node | nil,
+ [
+ { item_low_bound, hash_of_node | nil },
+ { item_low_bound, hash_of_node | nil },
+ ...
+ }
}
- }
"""
alias SData.PageStore, as: Store
diff --git a/shard/lib/data/merkletree.ex b/shard/lib/data/merkletree.ex
index 73679cf..94bd443 100644
--- a/shard/lib/data/merkletree.ex
+++ b/shard/lib/data/merkletree.ex
@@ -92,9 +92,17 @@ defmodule SData.MerkleTree do
end
@doc"""
- Get the hashes of all blocks in a range
+ Get the hashes of all blocks
"""
- def get_range(mt, range) do
- range |> Enum.map(&(get(mt, &1))) # TODO: do this efficiently
+ def get_all(mt) do
+ %Page{child_nblk: cn, list: list} = Store.get(mt.store, mt.root)
+ if cn == 1 do
+ list
+ else
+ list
+ |> Enum.map(&(%{mt | root: &1}))
+ |> Enum.map(&get_all/1)
+ |> Enum.reduce([], &(&2++&1))
+ end
end
end
diff --git a/shard/lib/data/signrev.ex b/shard/lib/data/signrev.ex
index 6360b53..164df03 100644
--- a/shard/lib/data/signrev.ex
+++ b/shard/lib/data/signrev.ex
@@ -56,7 +56,7 @@ defmodule SData.SignRev do
@doc"""
Check that a signed binary is correct and merge it into the SignRev.
- Returns {true, new_sr} if an update happenned, {false, sr} otherwise.
+ Returns `{true, new_sr}` if an update happenned, `{false, sr}` otherwise.
"""
def merge(sr, signed, pk) do
case Shard.Keys.open(pk, signed) do
diff --git a/shard/lib/data/store.ex b/shard/lib/data/store.ex
index ca12cd0..ce5618c 100644
--- a/shard/lib/data/store.ex
+++ b/shard/lib/data/store.ex
@@ -24,6 +24,10 @@ defprotocol SData.PageStore do
This protocol may also be implemented by store proxies that track
operations and implement different synchronization or caching mechanisms.
+
+ A page store is an object that stores data pages (arbitrary Erlang terms) and
+ identifies them by their hash. Dependencies may exist between pages, in which
+ case they form a Merkle DAG.
"""
@doc"""
@@ -60,8 +64,15 @@ end
defmodule SData.LocalStore do
+ @moduledoc"""
+ A page store that saves all pages locally in RAM. The store is basically a dictionnary
+ of hash to term mappings, which is mutated by put operations.
+ """
defstruct [:pages]
+ @doc"""
+ Create empty LocalStore.
+ """
def new() do
%SData.LocalStore{ pages: %{} }
end
diff --git a/shard/lib/keys.ex b/shard/lib/keys.ex
index 412baa2..3a97b5f 100644
--- a/shard/lib/keys.ex
+++ b/shard/lib/keys.ex
@@ -1,6 +1,6 @@
defmodule Shard.Keys do
@moduledoc"""
- Module for saving private keys.
+ Module for saving private keys, signing messages and checking message signatures.
"""
use Agent
@@ -39,6 +39,10 @@ defmodule Shard.Keys do
:binary.longest_common_suffix([pk, suffix]) == byte_size(suffix)
end
+ @doc"""
+ Return any public key for which we have the secret key. Generates a new keypair
+ if necessary.
+ """
def get_any_identity() do
Agent.get(__MODULE__, fn _ ->
case list_identities() do
@@ -96,6 +100,9 @@ defmodule Shard.Keys do
end
end
+ @doc"""
+ Check if we have the secret key associated with a public key.
+ """
def have_sk?(pk) do
case :dets.lookup @key_db, pk do
[{^pk, _sk}] -> true
@@ -103,6 +110,9 @@ defmodule Shard.Keys do
end
end
+ @doc"""
+ Return the secret key associated with a public key if we have it or `nil` otherwise.
+ """
def get_sk(pk) do
case :dets.lookup @key_db, pk do
[{^pk, sk}] -> sk
@@ -111,12 +121,12 @@ defmodule Shard.Keys do
end
@doc"""
- Lookup the secret key for a pk and generate a detached signature for a message.
+ Lookup the secret key for a pk and generate a detached signature for a message.
- The original message is not returned.
+ The original message is not returned.
- Answer is {:ok, signature} if it worked, or :not_found if we didn't find the key.
-
+ Answer is {:ok, signature} if it worked, or :not_found if we don't have the corresponding
+ secret key.
"""
def sign_detached(pk, bin) do
case :dets.lookup @key_db, pk do
@@ -127,9 +137,9 @@ defmodule Shard.Keys do
end
@doc"""
- Verify a detached signature for a message
+ Verify a detached signature for a message
- Returns :ok if the signature was correct.
+ Returns :ok if the signature was correct.
"""
def verify(pk, bin, sign) do
if valid_identity_pk? pk do
@@ -143,12 +153,16 @@ defmodule Shard.Keys do
end
@doc"""
- Check if a public key is a valid identity pk. Requirement: have the correct suffix.
+ Check if a public key is a valid identity pk. Requirement: have the correct suffix.
"""
def valid_identity_pk?(pk) do
check_suffix(pk, Application.get_env(:shard, :identity_suffix))
end
+ @doc"""
+ Creates a displayable representation of a public key by taking the hex representation
+ of its first four bytes. (not tamper proof but better than nothing)
+ """
def pk_display(pk) do
pk
|> binary_part(0, 4)
diff --git a/shard/lib/manager.ex b/shard/lib/manager.ex
index c3897a3..ed21380 100644
--- a/shard/lib/manager.ex
+++ b/shard/lib/manager.ex
@@ -14,7 +14,8 @@ defprotocol Shard.Manifest do
"""
@doc"""
- Get the module in question.
+ Get the module that implements the shard. All shard modules must have a function
+ `start_link` that start the shard process and take a single argument: the manifest.
"""
def module(manifest)
@@ -26,29 +27,38 @@ end
defmodule Shard.Manager do
@moduledoc"""
- Maintains several important tables :
+ The manager is the main process by which shards are started, stopped, and their lifetime
+ monitored.
- - :shard_db (persistent with DETS)
-
- List of
- { id, manifest, why_have_it, state }
+ Maintains several important tables :
- why_have_it := {:pinned, %MapSet{who requires it...}, %MapSet{who it requires...}}
- | {:req, %MapSet{who requires it...}, %MapSet{who it requires...}}
- | {:cached, expiry_date}
+ - `@shard_db` (persistent with DETS), a list of:
+
+ ```
+ { id, manifest, why_have_it, state }
+
+ why_have_it := {:pinned, %MapSet{who requires it...}, %MapSet{who it requires...}}
+ | {:req, %MapSet{who requires it...}, %MapSet{who it requires...}}
+ | {:cached, expiry_date}
+ ```
+
+ - `@peer_db` (persistent with DETS), a multi-list of:
- - :peer_db (persistent with DETS)
+ ```
+ { shard_id, peer_info } # TODO: add health info (last seen, ping, etc)
- Mult-list of
- { shard_id, peer_info } # TODO: add health info (last seen, ping, etc)
+ peer_info := {:inet, ip, port}
+ TODO peer_info |= {:inet6, ip, port} | {:onion, name}
+ ```
- peer_info := {:inet, ip, port}
- TODO peer_info |= {:inet6, ip, port} | {:onion, name}
+ - `:shard_procs` (not persistent), a list of:
- - :shard_procs (not persistent)
+ ```
+ { {id, path}, pid }
+ ```
- List of
- { {id, path}, pid }
+ The path value is used to distinguish between a shard's main process (`path == nil`)
+ and companion sub-processes such as a page store used by the shard.
"""
use GenServer
diff --git a/shard/lib/net/addr.ex b/shard/lib/net/addr.ex
index c1d2f05..b92ae70 100644
--- a/shard/lib/net/addr.ex
+++ b/shard/lib/net/addr.ex
@@ -1,4 +1,10 @@
defmodule SNet.Addr do
+ @moduledoc"""
+ Helper module for getting our IP addresses.
+
+ Runs an agent that gets our public IPv4 address on the internet and stores it.
+ """
+
use Agent
require Logger
@@ -21,6 +27,9 @@ defmodule SNet.Addr do
end
end
+ @doc"""
+ Reteurn the list of IPv4 address for our network interfaces.
+ """
def get_if_inet4 do
{:ok, ifs} = :inet.getifaddrs
for {_, opts} <- ifs,
@@ -32,15 +41,24 @@ defmodule SNet.Addr do
end
end
+ @doc"""
+ Return our public IPv4 address as observed by an external API provider (`ipify.org`)
+ """
def get_pub_inet4 do
Agent.get(__MODULE__, &(&1))
end
+ @doc"""
+ Get all our IPv4 addresses.
+ """
def get_all_inet4 do
addrset = for x <- get_if_inet4() ++ get_pub_inet4(), into: %MapSet{}, do: x
MapSet.to_list addrset
end
+ @doc"""
+ Determines if an IP address is ours or not.
+ """
def is_local?({:inet, ip, port}) do
port == Application.get_env(:shard, :port) and (ip == {127,0,0,1} or ip in get_all_inet4())
end
diff --git a/shard/lib/net/auth.ex b/shard/lib/net/auth.ex
index c903093..186b506 100644
--- a/shard/lib/net/auth.ex
+++ b/shard/lib/net/auth.ex
@@ -1,3 +1,13 @@
defmodule SNet.Auth do
+ @moduledoc"""
+ Structure for auth values that define if a connection is with an anonymous
+ peer or with an authenticated peer.
+
+ Message handlers in shards will receive an `auth` parameter equal to `nil` if the
+ connection where the message comes from is not authenticated, or
+ `%SNet.Auth{my_pk: my_pk, his_pk: his_pk}` in the case where the connection is authenticated,
+ we are known to them as `my_pk` and they are known to us as `his_pk`.
+ """
+
defstruct [:my_pk, :his_pk]
end
diff --git a/shard/lib/net/group.ex b/shard/lib/net/group.ex
index a5f0867..f3d5962 100644
--- a/shard/lib/net/group.ex
+++ b/shard/lib/net/group.ex
@@ -25,11 +25,17 @@ defprotocol SNet.Group do
@doc"""
Check if a peer is allowed to participate in this group.
+ The `auth` parameter is `nil` or a `SNet.Auth` struct.
"""
def in_group?(group, conn_pid, auth)
end
defmodule SNet.PubShardGroup do
+ @moduledoc"""
+ A network group defined as all the people interested in a given shard.
+
+ %SNet.PubShardGroup{id: shard_id}
+ """
defstruct [:id]
defimpl SNet.Group do
@@ -85,6 +91,12 @@ defmodule SNet.PubShardGroup do
end
defmodule SNet.PrivGroup do
+ @moduledoc"""
+ A private networking group defined by the list of public keys of people allowed to
+ participate.
+
+ %SNet.PrivGroup{pk_list: [pk1, pk2, ...]}
+ """
defstruct [:pk_list]
defimpl SNet.Group do
diff --git a/shard/lib/net/manager.ex b/shard/lib/net/manager.ex
index fb92f13..e4d8ad9 100644
--- a/shard/lib/net/manager.ex
+++ b/shard/lib/net/manager.ex
@@ -1,9 +1,8 @@
defmodule SNet.Manager do
@moduledoc"""
- - :connections (not persistent)
+ Maintains a table `:connections` of currently connected peers, which is a list of:
- List of
- { peer_info, pid, nil | {my_pk, his_pk} }
+ { peer_info, pid, nil | %SNet.Auth{my_pk: my_pk, his_pk: his_pk} }
"""
use GenServer
@@ -91,7 +90,7 @@ defmodule SNet.Manager do
@doc"""
Connect to a peer specified by ip address and port
- peer_info := {:inet, ip, port}
+ peer_info := {:inet, ip, port}
"""
def add_peer(peer_info, opts \\ []) do
GenServer.call(__MODULE__, {:add_peer, peer_info, opts[:auth], opts[:callback]})
@@ -134,6 +133,11 @@ defmodule SNet.Manager do
end
end
+ @doc"""
+ Send message to a peer specified by peer info over authenticated channel.
+ `auth` is a `SNet.Auth` struct describing the required authentication.
+ Opens a connection if necessary.
+ """
def send_auth(peer_info, auth, msg) do
case :ets.match(:connections, {peer_info, :'$1', auth, :_}) do
[[pid]|_] ->
diff --git a/shard/lib/net/tcpconn.ex b/shard/lib/net/tcpconn.ex
index 21d25df..dc33bff 100644
--- a/shard/lib/net/tcpconn.ex
+++ b/shard/lib/net/tcpconn.ex
@@ -1,10 +1,9 @@
defmodule SNet.TCPConn do
@moduledoc"""
Secret handshake as described in this document:
- https://ssbc.github.io/scuttlebutt-protocol-guide/#peer-connections
+ <https://ssbc.github.io/scuttlebutt-protocol-guide/#peer-connections>
- Does not implement the stream protocol, we don't hide the length of packets.
- (TODO ^)
+ TODO: Does not implement the stream protocol, we don't hide the length of packets.
"""
@@ -18,15 +17,15 @@ defmodule SNet.TCPConn do
Expected initial state: a dict with the following keys:
- - socket: the socket
- - is_client: true if we are the initiator of the connection, false otherwise
- - my_port: if we are the client, what port should the other dial to recontact us
+ - `socket`: the socket, if we are responding to a connection
+ - `connect_to`: the IP and port we want to connect to, if we are the initiator of the connection
+ - `my_port`: if we are the initiator, what port should the other dial to recontact us
Optionnally, and only if we are the initiator of the connection, the following key:
- - auth: nil | {my_pk, list_accepted_his_pk}
+ - `auth`: `nil | {my_pk, list_accepted_his_pk}`
- If we are initiator of the connection, we will use crypto if and only if auth is not nil.
+ If we are initiator of the connection, we will use crypto if and only if auth is not `nil`.
"""
def start_link(state) do
GenServer.start_link(__MODULE__, state)
diff --git a/shard/lib/net/tcpserver.ex b/shard/lib/net/tcpserver.ex
index d7326ad..827fefa 100644
--- a/shard/lib/net/tcpserver.ex
+++ b/shard/lib/net/tcpserver.ex
@@ -1,4 +1,8 @@
defmodule SNet.TCPServer do
+ @moduledoc"""
+ Process for accepting TCP connections from peers.
+ """
+
require Logger
use Task, restart: :permanent
diff --git a/shard/lib/shard_uri.ex b/shard/lib/shard_uri.ex
index 1b186d2..81b4ab1 100644
--- a/shard/lib/shard_uri.ex
+++ b/shard/lib/shard_uri.ex
@@ -1,8 +1,12 @@
defmodule ShardURI do
@moduledoc"""
- Convert Shard manifests to and from text strings.
+ Convert Shard manifests to and from text strings. Not used by the shard library
+ internally, only provided for convenience.
"""
+ @doc"""
+ Get URI corresponding to shard manifest.
+ """
def from_manifest(m) do
case m do
%SApp.Chat.Manifest{channel: chan} -> "shard:chat:#{chan}"
@@ -19,6 +23,9 @@ defmodule ShardURI do
end
end
+ @doc"""
+ Parse URI and return corresponding manifest.
+ """
def to_manifest(p) do
case p do
"shard:chat:" <> chan ->