Merge branch 'feature/pinner' into 'develop'

Feature/pinner

See merge request ecentrics/concordia!12

packages/concordia-app/src/options/breezeOptions.js

@@ -8,6 +8,8 @@ import {
 const REACT_APP_RENDEZVOUS_HOST = process.env.REACT_APP_RENDEZVOUS_HOST || REACT_APP_RENDEZVOUS_HOST_DEFAULT;
 const REACT_APP_RENDEZVOUS_PORT = process.env.REACT_APP_RENDEZVOUS_PORT || REACT_APP_RENDEZVOUS_PORT_DEFAULT;
 
+export const RENDEZVOUS_URL = `http://${REACT_APP_RENDEZVOUS_HOST}:${REACT_APP_RENDEZVOUS_PORT}`;
+
 const breezeOptions = {
   ipfs: {
     repo: 'concordia',

packages/concordia-contracts/contracts/Forum.sol

@@ -18,7 +18,8 @@ contract Forum {
     }
 
     mapping(address => User) users;
-    mapping(string => address) userAddresses;
+    mapping(string => address) usernameAddresses;
+    address[] userAddresses;
 
     event UserSignedUp(string username, address userAddress);
     event UsernameUpdated(string newName, string oldName, address userAddress);
@@ -27,7 +28,8 @@ contract Forum {
         require(!hasUserSignedUp(msg.sender), USER_HAS_NOT_SIGNED_UP);
         require(!isUserNameTaken(username), USERNAME_TAKEN);
         users[msg.sender] = User(username, new uint[](0), new uint[](0), block.timestamp, true);
-        userAddresses[username] = msg.sender;
+        usernameAddresses[username] = msg.sender;
+        userAddresses.push(msg.sender);
         emit UserSignedUp(username, msg.sender);
         return true;
     }
@@ -36,9 +38,9 @@ contract Forum {
         require(hasUserSignedUp(msg.sender), USER_HAS_NOT_SIGNED_UP);
         require(!isUserNameTaken(newUsername), USERNAME_TAKEN);
         string memory oldUsername = getUsername(msg.sender);
-        delete userAddresses[users[msg.sender].username];
+        delete usernameAddresses[users[msg.sender].username];
         users[msg.sender].username = newUsername;
-        userAddresses[newUsername] = msg.sender;
+        usernameAddresses[newUsername] = msg.sender;
         emit UsernameUpdated(newUsername, oldUsername, msg.sender);
         return true;
     }
@@ -49,7 +51,7 @@ contract Forum {
     }
 
     function getUserAddress(string memory username) public view returns (address) {
-        return userAddresses[username];
+        return usernameAddresses[username];
     }
 
     function hasUserSignedUp(address userAddress) public view returns (bool) {
@@ -82,6 +84,10 @@ contract Forum {
         return users[userAddress];
     }
 
+    function getUserAddresses() public view returns (address[] memory) {
+        return userAddresses;
+    }
+
     //----------------------------------------POSTING----------------------------------------
     struct Topic {
         uint topicID;

packages/concordia-contracts/package.json

@@ -16,8 +16,8 @@
     "upload": "node ./utils/contractsProviderUtils.js ${npm_package_version}-dev latest"
   },
   "dependencies": {
-    "@openzeppelin/contracts": "~3.2.0",
-    "truffle": "~5.1.55",
+    "@openzeppelin/contracts": "~3.3.0",
+    "truffle": "~5.1.58",
     "unirest": "^0.6.0"
   },
   "devDependencies": {

packages/concordia-pinner/.eslintrc.js

@@ -0,0 +1,60 @@
+module.exports = {
+  env: {
+    browser: true,
+    es6: true,
+    jest: true,
+  },
+  extends: [
+    'plugin:react/recommended',
+    'airbnb',
+  ],
+  globals: {
+    Atomics: 'readonly',
+    SharedArrayBuffer: 'readonly',
+  },
+  parser: 'babel-eslint',
+  parserOptions: {
+    ecmaFeatures: {
+      jsx: true,
+    },
+    ecmaVersion: 2018,
+    sourceType: 'module',
+  },
+  plugins: [
+    'react',
+    'react-hooks',
+  ],
+  rules: {
+    'react/jsx-props-no-spreading': 'off',
+    'import/extensions': 'off',
+    'react/jsx-indent': [
+      'error',
+      4,
+      {
+        checkAttributes: true,
+        indentLogicalExpressions: true,
+      },
+    ],
+    'react/require-default-props': 'off',
+    'react/prop-types': 'off',
+    'react-hooks/rules-of-hooks': 'error',
+    'react-hooks/exhaustive-deps': 'error',
+    'max-len': ['warn', { code: 120, tabWidth: 4 }],
+    'no-unused-vars': 'warn',
+    'no-console': 'off',
+    'no-shadow': 'warn',
+    'no-multi-str': 'warn',
+    'jsx-a11y/label-has-associated-control': [2, {
+      labelAttributes: ['label'],
+      controlComponents: ['Input'],
+      depth: 3,
+    }],
+  },
+  settings: {
+    'import/resolver': {
+      node: {
+        extensions: ['.js', '.jsx'],
+      },
+    },
+  },
+};

packages/concordia-pinner/.gitattributes

@@ -0,0 +1,10 @@
+# Set the default behavior, in case people don't have core.autocrlf set.
+* text=auto eol=lf
+
+# Denote all files that are truly binary and should not be modified.
+*.png binary
+*.jpg binary
+*.ico binary
+
+# Solidity
+*.sol linguist-language=Solidity

packages/concordia-pinner/.gitignore

@@ -0,0 +1,40 @@
+# Node
+/node_modules
+packages/*/node_modules
+packages/concordia-contracts/build
+
+# IDE
+.DS_Store
+.idea
+
+# Build Directories
+/build
+/src/build
+/packages/concordia-app/build
+/packages/concordia-contracts/build
+
+# Logs
+/log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# Docker volumes
+docker/volumes
+docker/ganache/volumes
+docker/reports
+
+# Env var files
+docker/env/concordia.env
+docker/env/contracts.env
+
+# Misc
+.env.local
+.env.development.local
+.env.test.local
+.env.production.local
+
+# IPFS & OrbitDB Storage
+ipfs
+orbitdb
+

packages/concordia-pinner/package.json

@@ -0,0 +1,38 @@
+{
+  "name": "concordia-pinner",
+  "description": "An OrbitDB pinning service for Concordia.",
+  "version": "0.1.0",
+  "private": true,
+  "main": "src/index.js",
+  "scripts": {
+    "start": "node -r esm src/index.js",
+    "clean": "rimraf ipfs orbitdb"
+  },
+  "license": "MIT",
+  "dependencies": {
+    "@ezerous/eth-identity-provider": "~0.1.2",
+    "concordia-app": "~0.1.0",
+    "concordia-contracts": "~0.1.0",
+    "esm": "~3.2.25",
+    "express": "^4.17.1",
+    "ipfs": "~0.52.1",
+    "is-reachable": "^5.0.0",
+    "level": "~6.0.1",
+    "libp2p": "~0.30.0",
+    "libp2p-bootstrap": "~0.12.1",
+    "libp2p-gossipsub": "~0.8.0",
+    "libp2p-kad-dht": "~0.20.1",
+    "libp2p-mdns": "~0.15.0",
+    "libp2p-mplex": "~0.10.0",
+    "libp2p-noise": "~2.0.1",
+    "libp2p-tcp": "~0.15.1",
+    "libp2p-webrtc-star": "~0.20.2",
+    "lodash": "^4.17.20",
+    "orbit-db": "~0.26.0",
+    "orbit-db-identity-provider": "~0.3.1",
+    "rimraf": "~3.0.2",
+    "web3": "~1.3.0",
+    "web3-eth-contract": "^1.3.1",
+    "wrtc": "~0.4.6"
+  }
+}

packages/concordia-pinner/src/app.js

@@ -0,0 +1,61 @@
+import express from 'express';
+import _ from 'lodash';
+import isReachable from 'is-reachable';
+import { API_PORT, RENDEZVOUS_URL, WEB3_PROVIDER_URL } from './constants';
+
+const POLLING_INTERVAL = 1000;
+
+const responseBody = {
+  ipfs: {
+    id: '', localAddresses: [], peers: [], totalPeers: 0, repoStats: {},
+  },
+  orbit: { identity: {}, databases: [] },
+  web3: { url: WEB3_PROVIDER_URL, reachable: false },
+  rendezvous: { url: RENDEZVOUS_URL, reachable: false },
+  timestamp: 0,
+};
+
+async function getStats(orbit) {
+  try {
+    // eslint-disable-next-line no-underscore-dangle
+    const ipfs = orbit._ipfs;
+    const { id } = await ipfs.id();
+    const peers = await ipfs.swarm.peers();
+    const localAddresses = await ipfs.swarm.localAddrs();
+    const repoStats = await ipfs.stats.repo();
+    const uniquePeers = _.uniqBy(peers, 'peer');
+    const orbitIdentity = orbit.identity;
+    const databases = Object.keys(orbit.stores);
+    const isWeb3Reachable = await isReachable(WEB3_PROVIDER_URL);
+    const isRendezvousReachable = await isReachable(RENDEZVOUS_URL);
+    const timestamp = +new Date();
+
+    responseBody.ipfs.id = id;
+    responseBody.ipfs.peers = uniquePeers;
+    responseBody.ipfs.totalPeers = uniquePeers.length;
+    responseBody.ipfs.localAddresses = localAddresses;
+    responseBody.ipfs.repoStats = repoStats;
+    responseBody.orbit.identity = orbitIdentity;
+    responseBody.orbit.databases = databases;
+    responseBody.web3.reachable = isWeb3Reachable;
+    responseBody.rendezvous.reachable = isRendezvousReachable;
+    responseBody.timestamp = timestamp;
+  } catch (err) {
+    console.error('Error while getting stats:', err);
+  }
+}
+
+const startAPI = (orbit) => {
+  const app = express();
+  app.get('/', async (req, res) => {
+    res.send(responseBody);
+  });
+
+  app.listen(API_PORT, () => {
+    console.log(`Pinner API at http://localhost:${API_PORT}!`);
+  });
+
+  setInterval(getStats, POLLING_INTERVAL, orbit);
+};
+
+export default startAPI;

packages/concordia-pinner/src/constants.js

@@ -0,0 +1,17 @@
+import breezeOptions, { RENDEZVOUS_URL } from 'concordia-app/src/options/breezeOptions';
+import { WEB3_HOST_DEFAULT, WEB3_PORT_DEFAULT } from 'concordia-app/src/constants/configuration/defaults';
+import path from 'path';
+
+const { WEB3_HOST, WEB3_PORT } = process.env;
+
+const API_PORT = process.env.PINNER_API_PORT || 4444;
+
+const WEB3_PROVIDER_URL = (WEB3_HOST !== undefined && WEB3_PORT !== undefined)
+  ? `ws://${WEB3_HOST}:${WEB3_PORT}`
+  : `ws://${WEB3_HOST_DEFAULT}:${WEB3_PORT_DEFAULT}`;
+
+export const swarmAddresses = breezeOptions.ipfs.config.Addresses.Swarm;
+
+export const ORBIT_DIRECTORY_DEFAULT = path.join(__dirname, '..', 'orbitdb');
+
+export { API_PORT, WEB3_PROVIDER_URL, RENDEZVOUS_URL };

packages/concordia-pinner/src/index.js

@@ -0,0 +1,80 @@
+import Web3 from 'web3';
+import Contract from 'web3-eth-contract';
+import IPFS from 'ipfs';
+import { contracts } from 'concordia-contracts';
+import { FORUM_CONTRACT } from 'concordia-app/src/constants/contracts/ContractNames';
+import { createOrbitInstance, getPeerDatabases, openKVDBs } from './utils/orbitUtils';
+import ipfsOptions from './options/ipfsOptions';
+import { WEB3_PROVIDER_URL } from './constants';
+import startAPI from './app';
+
+process.on('unhandledRejection', (error) => {
+  // This happens when attempting to initialize without any available Swarm addresses (e.g. Rendezvous)
+  if (error.code === 'ERR_NO_VALID_ADDRESSES') {
+    console.error('unhandledRejection', error.message);
+    process.exit(1);
+  }
+
+  // Don't swallow other errors
+  console.error(error);
+  throw error;
+});
+
+const getDeployedContract = async (web3) => {
+  const forumContract = contracts.find((contract) => contract.contractName === FORUM_CONTRACT);
+
+  return web3.eth.net.getId()
+    .then((networkId) => forumContract.networks[networkId].address)
+    .then((contractAddress) => {
+      Contract.setProvider(WEB3_PROVIDER_URL);
+      const contract = new Contract(forumContract.abi, contractAddress);
+
+      return { contract, contractAddress };
+    });
+};
+
+// Open & replicate databases of existing users
+const openExistingUsersDatabases = async (contract, orbit) => contract.methods.getUserAddresses().call()
+  .then((userAddresses) => getPeerDatabases(orbit, userAddresses))
+  .then((peerDBs) => openKVDBs(orbit, peerDBs));
+
+const handleWeb3LogEvent = (web3, eventJsonInterface, orbit) => (error, result) => {
+  if (!error) {
+    const eventObj = web3.eth.abi.decodeLog(
+      eventJsonInterface.inputs,
+      result.data,
+      result.topics.slice(1),
+    );
+    const userAddress = eventObj[1];
+    console.log('User signed up:', userAddress);
+    getPeerDatabases(orbit, [userAddress])
+      .then((peerDBs) => openKVDBs(orbit, peerDBs));
+  }
+};
+
+const main = async () => {
+  console.log('Initializing...');
+  const web3 = new Web3(new Web3.providers.WebsocketProvider(WEB3_PROVIDER_URL));
+
+  getDeployedContract(web3)
+    .then(({ contract, contractAddress }) => IPFS.create(ipfsOptions)
+      .then((ipfs) => createOrbitInstance(ipfs, contractAddress))
+      .then((orbit) => openExistingUsersDatabases(contract, orbit)
+        .then(() => {
+          // Listen for new users and subscribe to their databases
+          const eventJsonInterface = web3.utils._.find(
+            // eslint-disable-next-line no-underscore-dangle
+            contract._jsonInterface,
+            (obj) => obj.name === 'UserSignedUp' && obj.type === 'event',
+          );
+
+          web3.eth.subscribe('logs', {
+            address: contractAddress,
+            topics: [eventJsonInterface.signature],
+          }, handleWeb3LogEvent(web3, eventJsonInterface, orbit));
+
+          startAPI(orbit);
+        })));
+};
+
+main();

packages/concordia-pinner/src/options/ipfsOptions.js

@@ -0,0 +1,22 @@
+import libp2pBundle from './libp2pBundle';
+import { swarmAddresses } from '../constants';
+
+export default {
+  repo: 'ipfs',
+  config: {
+    Profile: 'server',
+    Addresses: {
+      Swarm: swarmAddresses,
+    },
+  },
+  libp2p: libp2pBundle,
+  EXPERIMENTAL: {
+    pubsub: true,
+  },
+  preload: {
+    enabled: false,
+  },
+  init: {
+    emptyRepo: true,
+  },
+};

packages/concordia-pinner/src/options/libp2pBundle.js

@@ -0,0 +1,90 @@
+import Libp2p from 'libp2p';
+import wrtc from 'wrtc';
+import MulticastDNS from 'libp2p-mdns';
+import WebrtcStar from 'libp2p-webrtc-star';
+import Bootstrap from 'libp2p-bootstrap';
+import Gossipsub from 'libp2p-gossipsub';
+import KadDHT from 'libp2p-kad-dht';
+import MPLEX from 'libp2p-mplex';
+import { NOISE } from 'libp2p-noise';
+import { swarmAddresses } from '../constants';
+
+// See also: https://github.com/libp2p/js-libp2p/blob/master/doc/CONFIGURATION.md
+export default (opts) => new Libp2p({
+  peerId: opts.peerId,
+  addresses: {
+    listen: swarmAddresses,
+  },
+  connectionManager: {
+    minPeers: 25,
+    maxPeers: 100,
+    pollInterval: 5000,
+  },
+  modules: {
+    transport: [
+      WebrtcStar,
+    ],
+    streamMuxer: [
+      MPLEX,
+    ],
+    connEncryption: [
+      NOISE,
+    ],
+    peerDiscovery: [
+      MulticastDNS,
+      Bootstrap,
+    ],
+    dht: KadDHT,
+    pubsub: Gossipsub,
+  },
+  config: {
+    transport: {
+      [WebrtcStar.prototype[Symbol.toStringTag]]: {
+        wrtc,
+      },
+    },
+    peerDiscovery: {
+      autoDial: true,
+      mdns: {
+        enabled: true,
+        interval: 10000,
+      },
+      bootstrap: {
+        enabled: true,
+        interval: 30e3,
+        list: opts.config.Bootstrap,
+      },
+    },
+    relay: {
+      enabled: true,
+      hop: {
+        enabled: true,
+        active: true,
+      },
+    },
+    dht: {
+      enabled: true,
+      kBucketSize: 20,
+      randomWalk: {
+        enabled: true,
+        interval: 10e3,
+        timeout: 2e3,
+      },
+    },
+    pubsub: {
+      enabled: true,
+      emitself: true,
+    },
+  },
+  metrics: {
+    enabled: true,
+    computeThrottleMaxQueueSize: 1000,
+    computeThrottleTimeout: 2000,
+    movingAverageIntervals: [
+      60 * 1000, // 1 minute
+      5 * 60 * 1000, // 5 minutes
+      15 * 60 * 1000, // 15 minutes
+    ],
+    maxOldPeersRetention: 50,
+  },
+});

packages/concordia-pinner/src/utils/orbitUtils.js

@@ -0,0 +1,46 @@
+import OrbitDB from 'orbit-db';
+import Identities from 'orbit-db-identity-provider';
+import { EthereumContractIdentityProvider } from '@ezerous/eth-identity-provider';
+import Web3 from 'web3';
+import { ORBIT_DIRECTORY_DEFAULT } from '../constants';
+
+// TODO: share code below with frontend (?)
+const determineDBAddress = async ({
+  orbit, dbName, type, identityId,
+}) => orbit.determineAddress(dbName, type, { accessController: { write: [identityId] } })
+  .then((orbitAddress) => {
+    const ipfsMultihash = orbitAddress.root;
+    return `/orbitdb/${ipfsMultihash}/${dbName}`;
+  });
+
+const determineKVAddress = async ({ orbit, dbName, userAddress }) => determineDBAddress({
+  orbit, dbName, type: 'keyvalue', identityId: userAddress + EthereumContractIdentityProvider.contractAddress,
+});
+
+export const createOrbitInstance = async (ipfs, contractAddress) => {
+  Identities.addIdentityProvider(EthereumContractIdentityProvider);
+
+  EthereumContractIdentityProvider.setWeb3(new Web3()); // We need a fully-featured new Web3 for signature verification
+  EthereumContractIdentityProvider.setContractAddress(contractAddress);
+
+  const ORBIT_DIRECTORY = process.env.ORBIT_DIRECTORY || ORBIT_DIRECTORY_DEFAULT;
+
+  return OrbitDB.createInstance(ipfs, { directory: ORBIT_DIRECTORY });
+};
+
+export const getPeerDatabases = async (orbit, userAddresses) => Promise.all(userAddresses
+  .flatMap((userAddress) => [
+    determineKVAddress({ orbit, dbName: 'user', userAddress }),
+    determineKVAddress({ orbit, dbName: 'posts', userAddress }),
+    determineKVAddress({ orbit, dbName: 'topics', userAddress }),
+  ]));
+
+export const openKVDBs = async (orbit, databases) => {
+  databases
+    .forEach((database) => {
+      orbit
+        .keyvalue(database)
+        .then((store) => store.events.on('replicated', (address) => console.log(`Replicated ${address}`)));
+      console.log(`Opened ${database}`);
+    });
+};