ECEntrics
/
concordia
mirror of https://gitlab.com/ecentrics/concordia
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

refactor: remove awaits, fix lint issues, add env vars for orbitdb path

develop
Apostolos Fanakis 4 years ago
parent
5d584afa19
commit
da5311afea
9 changed files with 743 additions and 339 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 60
      packages/concordia-pinner/.eslintrc.js
  2. 2
      packages/concordia-pinner/package.json
  3. 94
      packages/concordia-pinner/src/app.js
  4. 12
      packages/concordia-pinner/src/constants.js
  5. 106
      packages/concordia-pinner/src/index.js
  6. 35
      packages/concordia-pinner/src/options/ipfsOptions.js
  7. 140
      packages/concordia-pinner/src/options/libp2pBundle.js
  8. 75
      packages/concordia-pinner/src/utils/orbitUtils.js
  9. 558
      yarn.lock

60
packages/concordia-pinner/.eslintrc.js
View File

@ -0,0 +1,60 @@
module.exports = {
env: {
browser: true,
es6: true,
jest: true,
},
extends: [
'plugin:react/recommended',
'airbnb',
],
globals: {
Atomics: 'readonly',
SharedArrayBuffer: 'readonly',
},
parser: 'babel-eslint',
parserOptions: {
ecmaFeatures: {
jsx: true,
},
ecmaVersion: 2018,
sourceType: 'module',
},
plugins: [
'react',
'react-hooks',
],
rules: {
'react/jsx-props-no-spreading': 'off',
'import/extensions': 'off',
'react/jsx-indent': [
'error',
4,
{
checkAttributes: true,
indentLogicalExpressions: true,
},
],
'react/require-default-props': 'off',
'react/prop-types': 'off',
'react-hooks/rules-of-hooks': 'error',
'react-hooks/exhaustive-deps': 'error',
'max-len': ['warn', { code: 120, tabWidth: 4 }],
'no-unused-vars': 'warn',
'no-console': 'off',
'no-shadow': 'warn',
'no-multi-str': 'warn',
'jsx-a11y/label-has-associated-control': [2, {
labelAttributes: ['label'],
controlComponents: ['Input'],
depth: 3,
}],
},
settings: {
'import/resolver': {
node: {
extensions: ['.js', '.jsx'],
},
},
},
};

2
packages/concordia-pinner/package.json
View File

@ -11,6 +11,8 @@
"license": "MIT",
"dependencies": {
"@ezerous/eth-identity-provider": "~0.1.2",
"concordia-app": "~0.1.0",
"concordia-contracts": "~0.1.0",
"esm": "~3.2.25",
"express": "^4.17.1",
"ipfs": "~0.52.1",

94
packages/concordia-pinner/src/app.js
View File

@ -1,57 +1,61 @@
import express from 'express';
import _ from 'lodash';
import isReachable from 'is-reachable';
import { API_PORT, RENDEZVOUS_URL, WEB3_PROVIDER_URL } from './constants';
const POLLING_INTERVAL = 1000;
let app;
let responseBody = {
ipfs:{id:"", localAddresses:[], peers:[], totalPeers:0, repoStats:{}},
orbit:{identity:{}, databases:[]},
web3:{url:WEB3_PROVIDER_URL, reachable: false},
rendezvous:{url:RENDEZVOUS_URL, reachable: false},
timestamp:0
const responseBody = {
ipfs: {
id: '', localAddresses: [], peers: [], totalPeers: 0, repoStats: {},
},
orbit: { identity: {}, databases: [] },
web3: { url: WEB3_PROVIDER_URL, reachable: false },
rendezvous: { url: RENDEZVOUS_URL, reachable: false },
timestamp: 0,
};
export function startAPI(orbit){
app = express();
app.get('/', async (req, res) => {
res.send(responseBody);
});
async function getStats(orbit) {
try {
// eslint-disable-next-line no-underscore-dangle
const ipfs = orbit._ipfs;
const { id } = await ipfs.id();
const peers = await ipfs.swarm.peers();
const localAddresses = await ipfs.swarm.localAddrs();
const repoStats = await ipfs.stats.repo();
const uniquePeers = _.uniqBy(peers, 'peer');
const orbitIdentity = orbit.identity;
const databases = Object.keys(orbit.stores);
const isWeb3Reachable = await isReachable(WEB3_PROVIDER_URL);
const isRendezvousReachable = await isReachable(RENDEZVOUS_URL);
const timestamp = +new Date();
app.listen(API_PORT, () => {
console.log(`Pinner API at http://localhost:${API_PORT}!`);
});
setInterval(getStats, POLLING_INTERVAL, orbit);
responseBody.ipfs.id = id;
responseBody.ipfs.peers = uniquePeers;
responseBody.ipfs.totalPeers = uniquePeers.length;
responseBody.ipfs.localAddresses = localAddresses;
responseBody.ipfs.repoStats = repoStats;
responseBody.orbit.identity = orbitIdentity;
responseBody.orbit.databases = databases;
responseBody.web3.reachable = isWeb3Reachable;
responseBody.rendezvous.reachable = isRendezvousReachable;
responseBody.timestamp = timestamp;
} catch (err) {
console.error('Error while getting stats:', err);
}
}
async function getStats(orbit) {
try {
const ipfs = orbit._ipfs;
const {id} = await ipfs.id();
const peers = await ipfs.swarm.peers();
const localAddresses = await ipfs.swarm.localAddrs();
const repoStats = await ipfs.stats.repo();
const uniquePeers = _.uniqBy(peers, 'peer');
const orbitIdentity = orbit.identity;
const databases = Object.keys(orbit.stores);
const isWeb3Reachable = await isReachable(WEB3_PROVIDER_URL);
const isRendezvousReachable = await isReachable(RENDEZVOUS_URL);
const timestamp = + new Date();
responseBody.ipfs.id = id;
responseBody.ipfs.peers = uniquePeers;
responseBody.ipfs.totalPeers = uniquePeers.length;
responseBody.ipfs.localAddresses = localAddresses;
responseBody.ipfs.repoStats = repoStats;
responseBody.orbit.identity = orbitIdentity;
responseBody.orbit.databases = databases;
responseBody.web3.reachable = isWeb3Reachable;
responseBody.rendezvous.reachable = isRendezvousReachable;
responseBody.timestamp = timestamp;
} catch (err) {
console.error('Error while getting stats:', err)
}
}
const startAPI = (orbit) => {
const app = express();
app.get('/', async (req, res) => {
res.send(responseBody);
});
app.listen(API_PORT, () => {
console.log(`Pinner API at http://localhost:${API_PORT}!`);
});
setInterval(getStats, POLLING_INTERVAL, orbit);
};
export default startAPI;

12
packages/concordia-pinner/src/constants.js
View File

@ -1,13 +1,17 @@
import breezeOptions, {RENDEZVOUS_URL} from 'concordia-app/src/options/breezeOptions';
import breezeOptions, { RENDEZVOUS_URL } from 'concordia-app/src/options/breezeOptions';
import { WEB3_HOST_DEFAULT, WEB3_PORT_DEFAULT } from 'concordia-app/src/constants/configuration/defaults';
import path from 'path';
const { WEB3_HOST, WEB3_PORT } = process.env;
const API_PORT = process.env.PINNER_API_PORT || 4444;
const WEB3_PROVIDER_URL = (WEB3_HOST !== undefined && WEB3_PORT !== undefined)
? `ws://${WEB3_HOST}:${WEB3_PORT}`
: `ws://${WEB3_HOST_DEFAULT}:${WEB3_PORT_DEFAULT}`;
? `ws://${WEB3_HOST}:${WEB3_PORT}`
: `ws://${WEB3_HOST_DEFAULT}:${WEB3_PORT_DEFAULT}`;
export const swarmAddresses = breezeOptions.ipfs.config.Addresses.Swarm;
export {API_PORT, WEB3_PROVIDER_URL, RENDEZVOUS_URL};
export const ORBIT_DIRECTORY_DEFAULT = path.join(__dirname, '..', 'orbitdb');
export { API_PORT, WEB3_PROVIDER_URL, RENDEZVOUS_URL };

106
packages/concordia-pinner/src/index.js
View File

@ -1,60 +1,80 @@
import Web3 from 'web3';
import Contract from 'web3-eth-contract';
import IPFS from 'ipfs';
import { forumContract } from 'concordia-contracts';
import { contracts } from 'concordia-contracts';
import { FORUM_CONTRACT } from 'concordia-app/src/constants/contracts/ContractNames';
import { createOrbitInstance, getPeerDatabases, openKVDBs } from './utils/orbitUtils';
import ipfsOptions from './options/ipfsOptions';
import { WEB3_PROVIDER_URL } from './constants';
import { startAPI } from './app';
import startAPI from './app';
process.on('unhandledRejection', error => {
process.on('unhandledRejection', (error) => {
// This happens when attempting to initialize without any available Swarm addresses (e.g. Rendezvous)
if(error.code === 'ERR_NO_VALID_ADDRESSES'){
if (error.code === 'ERR_NO_VALID_ADDRESSES') {
console.error('unhandledRejection', error.message);
process.exit(1);
}
// Don't swallow other errors
console.error(error);
throw error;
});
async function main () {
const getDeployedContract = async (web3) => {
const forumContract = contracts.find((contract) => contract.contractName === FORUM_CONTRACT);
return web3.eth.net.getId()
.then((networkId) => forumContract.networks[networkId].address)
.then((contractAddress) => {
Contract.setProvider(WEB3_PROVIDER_URL);
const contract = new Contract(forumContract.abi, contractAddress);
return { contract, contractAddress };
});
};
// Open & replicate databases of existing users
const openExistingUsersDatabases = async (contract, orbit) => contract.methods.getUserAddresses().call()
.then((userAddresses) => getPeerDatabases(orbit, userAddresses))
.then((peerDBs) => openKVDBs(orbit, peerDBs));
const handleWeb3LogEvent = (web3, eventJsonInterface, orbit) => (error, result) => {
if (!error) {
const eventObj = web3.eth.abi.decodeLog(
eventJsonInterface.inputs,
result.data,
result.topics.slice(1),
);
const userAddress = eventObj[1];
console.log('User signed up:', userAddress);
getPeerDatabases(orbit, [userAddress])
.then((peerDBs) => openKVDBs(orbit, peerDBs));
}
};
const main = async () => {
console.log('Initializing...');
const web3 = new Web3(new Web3.providers.WebsocketProvider(WEB3_PROVIDER_URL));
const networkId = await web3.eth.net.getId();
const contractAddress = forumContract.networks[networkId].address;
Contract.setProvider(WEB3_PROVIDER_URL);
const contract = new Contract(forumContract.abi, contractAddress);
const ipfs = await IPFS.create(ipfsOptions);
const orbit = await createOrbitInstance(ipfs, contractAddress);
// Open & replicate databases of existing users
const userAddresses = await contract.methods.getUserAddresses().call();
const peerDBs = await getPeerDatabases(orbit, userAddresses);
await openKVDBs(orbit, peerDBs);
// Listen for new users and subscribe to their databases
const eventJsonInterface = web3.utils._.find(
contract._jsonInterface,
obj => obj.name === "UserSignedUp" && obj.type === 'event'
);
web3.eth.subscribe('logs', {
address: contractAddress,
topics: [eventJsonInterface.signature]
}, function(error, result){
if (!error) {
const eventObj = web3.eth.abi.decodeLog(
eventJsonInterface.inputs,
result.data,
result.topics.slice(1)
)
const userAddress = eventObj[1];
console.log(`User signed up:`, userAddress);
getPeerDatabases(orbit, [userAddress]).then(peerDBs => openKVDBs(orbit, peerDBs));
}
});
startAPI(orbit);
}
getDeployedContract(web3)
.then(({ contract, contractAddress }) => IPFS.create(ipfsOptions)
.then((ipfs) => createOrbitInstance(ipfs, contractAddress))
.then((orbit) => openExistingUsersDatabases(contract, orbit)
.then(() => {
// Listen for new users and subscribe to their databases
const eventJsonInterface = web3.utils._.find(
// eslint-disable-next-line no-underscore-dangle
contract._jsonInterface,
(obj) => obj.name === 'UserSignedUp' && obj.type === 'event',
);
web3.eth.subscribe('logs', {
address: contractAddress,
topics: [eventJsonInterface.signature],
}, handleWeb3LogEvent(web3, eventJsonInterface, orbit));
startAPI(orbit);
})));
};
main();

35
packages/concordia-pinner/src/options/ipfsOptions.js
View File

@ -1,21 +1,22 @@
import libp2pBundle from './libp2pBundle'
import libp2pBundle from './libp2pBundle';
import { swarmAddresses } from '../constants';
export default {
repo: 'ipfs',
config: {
Addresses: {
Swarm: swarmAddresses
},
repo: 'ipfs',
config: {
Profile: 'server',
Addresses: {
Swarm: swarmAddresses,
},
libp2p: libp2pBundle,
EXPERIMENTAL: {
pubsub: true,
},
preload: {
enabled: false,
},
init: {
emptyRepo: true,
},
}
},
libp2p: libp2pBundle,
EXPERIMENTAL: {
pubsub: true,
},
preload: {
enabled: false,
},
init: {
emptyRepo: true,
},
};

140
packages/concordia-pinner/src/options/libp2pBundle.js
View File

@ -10,83 +10,81 @@ import { NOISE } from 'libp2p-noise';
import { swarmAddresses } from '../constants';
// See also: https://github.com/libp2p/js-libp2p/blob/master/doc/CONFIGURATION.md
export default (opts) => {
return new Libp2p({
peerId: opts.peerId,
addresses: {
listen: swarmAddresses
},
connectionManager: {
minPeers: 25,
maxPeers: 100,
pollInterval: 5000
},
modules: {
transport: [
WebrtcStar
],
streamMuxer: [
MPLEX
],
connEncryption: [
NOISE
],
peerDiscovery: [
MulticastDNS,
Bootstrap
],
dht: KadDHT,
pubsub: Gossipsub
},
config: {
transport: {
[WebrtcStar.prototype[Symbol.toStringTag]]: {
wrtc
}
export default (opts) => new Libp2p({
peerId: opts.peerId,
addresses: {
listen: swarmAddresses,
},
connectionManager: {
minPeers: 25,
maxPeers: 100,
pollInterval: 5000,
},
modules: {
transport: [
WebrtcStar,
],
streamMuxer: [
MPLEX,
],
connEncryption: [
NOISE,
],
peerDiscovery: [
MulticastDNS,
Bootstrap,
],
dht: KadDHT,
pubsub: Gossipsub,
},
config: {
transport: {
[WebrtcStar.prototype[Symbol.toStringTag]]: {
wrtc,
},
peerDiscovery: {
autoDial: true,
mdns: {
enabled: true,
interval: 10000
},
bootstrap: {
enabled: true,
interval: 30e3,
list: opts.config.Bootstrap
}
},
peerDiscovery: {
autoDial: true,
mdns: {
enabled: true,
interval: 10000,
},
relay: {
bootstrap: {
enabled: true,
hop: {
enabled: true,
active: true
}
interval: 30e3,
list: opts.config.Bootstrap,
},
dht: {
},
relay: {
enabled: true,
hop: {
enabled: true,
kBucketSize: 20,
randomWalk: {
enabled: true,
interval: 10e3,
timeout: 2e3
}
active: true,
},
pubsub: {
},
dht: {
enabled: true,
kBucketSize: 20,
randomWalk: {
enabled: true,
emitself: true
}
interval: 10e3,
timeout: 2e3,
},
},
metrics: {
pubsub: {
enabled: true,
computeThrottleMaxQueueSize: 1000,
computeThrottleTimeout: 2000,
movingAverageIntervals: [
60 * 1000, // 1 minute
5 * 60 * 1000, // 5 minutes
15 * 60 * 1000 // 15 minutes
],
maxOldPeersRetention: 50
}
})
}
emitself: true,
},
},
metrics: {
enabled: true,
computeThrottleMaxQueueSize: 1000,
computeThrottleTimeout: 2000,
movingAverageIntervals: [
60 * 1000, // 1 minute
5 * 60 * 1000, // 5 minutes
15 * 60 * 1000, // 15 minutes
],
maxOldPeersRetention: 50,
},
});

75
packages/concordia-pinner/src/utils/orbitUtils.js
View File

@ -2,46 +2,45 @@ import OrbitDB from 'orbit-db';
import Identities from 'orbit-db-identity-provider';
import { EthereumContractIdentityProvider } from '@ezerous/eth-identity-provider';
import Web3 from 'web3';
export async function createOrbitInstance(ipfs, contractAddress){
Identities.addIdentityProvider(EthereumContractIdentityProvider);
EthereumContractIdentityProvider.setWeb3(new Web3()); // We need a fully-featured new Web3 for signature verification
EthereumContractIdentityProvider.setContractAddress(contractAddress);
return await OrbitDB.createInstance(ipfs);
}
export async function getPeerDatabases(orbit, userAddresses) {
const peerDBs = [];
for (const userAddress of userAddresses) {
peerDBs.push(await determineKVAddress({ orbit, dbName:'user', userAddress }));
peerDBs.push(await determineKVAddress({ orbit, dbName:'posts', userAddress }));
peerDBs.push(await determineKVAddress({ orbit, dbName:'topics', userAddress }));
}
return peerDBs;
}
export async function openKVDBs(orbit, databases) {
for (const db of databases){
const store = await orbit.keyvalue(db);
store.events.on('replicated', (address) => console.log(`Replicated ${address}`));
console.log(`Opened ${db}`);
}
}
import { ORBIT_DIRECTORY_DEFAULT } from '../constants';
// TODO: share code below with frontend (?)
async function determineDBAddress({
orbit, dbName, type, identityId,
}) {
const ipfsMultihash = (await orbit.determineAddress(dbName, type, {
accessController: { write: [identityId] },
})).root;
const determineDBAddress = async ({
orbit, dbName, type, identityId,
}) => orbit.determineAddress(dbName, type, { accessController: { write: [identityId] } })
.then((orbitAddress) => {
const ipfsMultihash = orbitAddress.root;
return `/orbitdb/${ipfsMultihash}/${dbName}`;
}
});
const determineKVAddress = async ({ orbit, dbName, userAddress }) => determineDBAddress({
orbit, dbName, type: 'keyvalue', identityId: userAddress + EthereumContractIdentityProvider.contractAddress,
});
export const createOrbitInstance = async (ipfs, contractAddress) => {
Identities.addIdentityProvider(EthereumContractIdentityProvider);
EthereumContractIdentityProvider.setWeb3(new Web3()); // We need a fully-featured new Web3 for signature verification
EthereumContractIdentityProvider.setContractAddress(contractAddress);
const ORBIT_DIRECTORY = process.env.ORBIT_DIRECTORY || ORBIT_DIRECTORY_DEFAULT;
return OrbitDB.createInstance(ipfs, { directory: ORBIT_DIRECTORY });
};
export const getPeerDatabases = async (orbit, userAddresses) => Promise.all(userAddresses
.flatMap((userAddress) => [
determineKVAddress({ orbit, dbName: 'user', userAddress }),
determineKVAddress({ orbit, dbName: 'posts', userAddress }),
determineKVAddress({ orbit, dbName: 'topics', userAddress }),
]));
async function determineKVAddress({ orbit, dbName, userAddress }) {
return determineDBAddress({
orbit, dbName, type: 'keyvalue', identityId: userAddress + EthereumContractIdentityProvider.contractAddress,
export const openKVDBs = async (orbit, databases) => {
databases
.forEach((database) => {
orbit
.keyvalue(database)
.then((store) => store.events.on('replicated', (address) => console.log(`Replicated ${address}`)));
console.log(`Opened ${database}`);
});
}
};

558
yarn.lock
View File

File diff suppressed because it is too large
Write Preview
Loading…
Cancel
Save