ECEntrics
/
concordia
mirror of https://gitlab.com/ecentrics/concordia
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

refactor: remove awaits, fix lint issues, add env vars for orbitdb path

develop
Apostolos Fanakis 4 years ago
parent
5d584afa19
commit
da5311afea
9 changed files with 743 additions and 339 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 60
      packages/concordia-pinner/.eslintrc.js
  2. 2
      packages/concordia-pinner/package.json
  3. 94
      packages/concordia-pinner/src/app.js
  4. 12
      packages/concordia-pinner/src/constants.js
  5. 106
      packages/concordia-pinner/src/index.js
  6. 35
      packages/concordia-pinner/src/options/ipfsOptions.js
  7. 140
      packages/concordia-pinner/src/options/libp2pBundle.js
  8. 75
      packages/concordia-pinner/src/utils/orbitUtils.js
  9. 558
      yarn.lock

60
packages/concordia-pinner/.eslintrc.js
View File

@ -0,0 +1,60 @@
module.exports = {
env: {
browser: true,
es6: true,
jest: true,
},
extends: [
'plugin:react/recommended',
'airbnb',
],
globals: {
Atomics: 'readonly',
SharedArrayBuffer: 'readonly',
},
parser: 'babel-eslint',
parserOptions: {
ecmaFeatures: {
jsx: true,
},
ecmaVersion: 2018,
sourceType: 'module',
},
plugins: [
'react',
'react-hooks',
],
rules: {
'react/jsx-props-no-spreading': 'off',
'import/extensions': 'off',
'react/jsx-indent': [
'error',
4,
{
checkAttributes: true,
indentLogicalExpressions: true,
},
],
'react/require-default-props': 'off',
'react/prop-types': 'off',
'react-hooks/rules-of-hooks': 'error',
'react-hooks/exhaustive-deps': 'error',
'max-len': ['warn', { code: 120, tabWidth: 4 }],
'no-unused-vars': 'warn',
'no-console': 'off',
'no-shadow': 'warn',
'no-multi-str': 'warn',
'jsx-a11y/label-has-associated-control': [2, {
labelAttributes: ['label'],
controlComponents: ['Input'],
depth: 3,
}],
},
settings: {
'import/resolver': {
node: {
extensions: ['.js', '.jsx'],
},
},
},
};

2
packages/concordia-pinner/package.json
View File

@ -11,6 +11,8 @@
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
"@ezerous/eth-identity-provider": "~0.1.2", "@ezerous/eth-identity-provider": "~0.1.2",
"concordia-app": "~0.1.0",
"concordia-contracts": "~0.1.0",
"esm": "~3.2.25", "esm": "~3.2.25",
"express": "^4.17.1", "express": "^4.17.1",
"ipfs": "~0.52.1", "ipfs": "~0.52.1",

94
packages/concordia-pinner/src/app.js
View File

@ -1,57 +1,61 @@
import express from 'express'; import express from 'express';
import _ from 'lodash'; import _ from 'lodash';
import isReachable from 'is-reachable'; import isReachable from 'is-reachable';
import { API_PORT, RENDEZVOUS_URL, WEB3_PROVIDER_URL } from './constants'; import { API_PORT, RENDEZVOUS_URL, WEB3_PROVIDER_URL } from './constants';
const POLLING_INTERVAL = 1000; const POLLING_INTERVAL = 1000;
let app; const responseBody = {
let responseBody = { ipfs: {
ipfs:{id:"", localAddresses:[], peers:[], totalPeers:0, repoStats:{}}, id: '', localAddresses: [], peers: [], totalPeers: 0, repoStats: {},
orbit:{identity:{}, databases:[]}, },
web3:{url:WEB3_PROVIDER_URL, reachable: false}, orbit: { identity: {}, databases: [] },
rendezvous:{url:RENDEZVOUS_URL, reachable: false}, web3: { url: WEB3_PROVIDER_URL, reachable: false },
timestamp:0 rendezvous: { url: RENDEZVOUS_URL, reachable: false },
timestamp: 0,
}; };
export function startAPI(orbit){ async function getStats(orbit) {
app = express(); try {
app.get('/', async (req, res) => { // eslint-disable-next-line no-underscore-dangle
res.send(responseBody); const ipfs = orbit._ipfs;
}); const { id } = await ipfs.id();
const peers = await ipfs.swarm.peers();
const localAddresses = await ipfs.swarm.localAddrs();
const repoStats = await ipfs.stats.repo();
const uniquePeers = _.uniqBy(peers, 'peer');
const orbitIdentity = orbit.identity;
const databases = Object.keys(orbit.stores);
const isWeb3Reachable = await isReachable(WEB3_PROVIDER_URL);
const isRendezvousReachable = await isReachable(RENDEZVOUS_URL);
const timestamp = +new Date();
app.listen(API_PORT, () => { responseBody.ipfs.id = id;
console.log(`Pinner API at http://localhost:${API_PORT}!`); responseBody.ipfs.peers = uniquePeers;
}); responseBody.ipfs.totalPeers = uniquePeers.length;
setInterval(getStats, POLLING_INTERVAL, orbit); responseBody.ipfs.localAddresses = localAddresses;
responseBody.ipfs.repoStats = repoStats;
responseBody.orbit.identity = orbitIdentity;
responseBody.orbit.databases = databases;
responseBody.web3.reachable = isWeb3Reachable;
responseBody.rendezvous.reachable = isRendezvousReachable;
responseBody.timestamp = timestamp;
} catch (err) {
console.error('Error while getting stats:', err);
}
} }
async function getStats(orbit) { const startAPI = (orbit) => {
try { const app = express();
const ipfs = orbit._ipfs; app.get('/', async (req, res) => {
const {id} = await ipfs.id(); res.send(responseBody);
const peers = await ipfs.swarm.peers(); });
const localAddresses = await ipfs.swarm.localAddrs();
const repoStats = await ipfs.stats.repo(); app.listen(API_PORT, () => {
const uniquePeers = _.uniqBy(peers, 'peer'); console.log(`Pinner API at http://localhost:${API_PORT}!`);
const orbitIdentity = orbit.identity; });
const databases = Object.keys(orbit.stores);
const isWeb3Reachable = await isReachable(WEB3_PROVIDER_URL); setInterval(getStats, POLLING_INTERVAL, orbit);
const isRendezvousReachable = await isReachable(RENDEZVOUS_URL); };
const timestamp = + new Date();
export default startAPI;
responseBody.ipfs.id = id;
responseBody.ipfs.peers = uniquePeers;
responseBody.ipfs.totalPeers = uniquePeers.length;
responseBody.ipfs.localAddresses = localAddresses;
responseBody.ipfs.repoStats = repoStats;
responseBody.orbit.identity = orbitIdentity;
responseBody.orbit.databases = databases;
responseBody.web3.reachable = isWeb3Reachable;
responseBody.rendezvous.reachable = isRendezvousReachable;
responseBody.timestamp = timestamp;
} catch (err) {
console.error('Error while getting stats:', err)
}
}

12
packages/concordia-pinner/src/constants.js
View File

@ -1,13 +1,17 @@
import breezeOptions, {RENDEZVOUS_URL} from 'concordia-app/src/options/breezeOptions'; import breezeOptions, { RENDEZVOUS_URL } from 'concordia-app/src/options/breezeOptions';
import { WEB3_HOST_DEFAULT, WEB3_PORT_DEFAULT } from 'concordia-app/src/constants/configuration/defaults'; import { WEB3_HOST_DEFAULT, WEB3_PORT_DEFAULT } from 'concordia-app/src/constants/configuration/defaults';
import path from 'path';
const { WEB3_HOST, WEB3_PORT } = process.env; const { WEB3_HOST, WEB3_PORT } = process.env;
const API_PORT = process.env.PINNER_API_PORT || 4444; const API_PORT = process.env.PINNER_API_PORT || 4444;
const WEB3_PROVIDER_URL = (WEB3_HOST !== undefined && WEB3_PORT !== undefined) const WEB3_PROVIDER_URL = (WEB3_HOST !== undefined && WEB3_PORT !== undefined)
? `ws://${WEB3_HOST}:${WEB3_PORT}` ? `ws://${WEB3_HOST}:${WEB3_PORT}`
: `ws://${WEB3_HOST_DEFAULT}:${WEB3_PORT_DEFAULT}`; : `ws://${WEB3_HOST_DEFAULT}:${WEB3_PORT_DEFAULT}`;
export const swarmAddresses = breezeOptions.ipfs.config.Addresses.Swarm; export const swarmAddresses = breezeOptions.ipfs.config.Addresses.Swarm;
export {API_PORT, WEB3_PROVIDER_URL, RENDEZVOUS_URL}; export const ORBIT_DIRECTORY_DEFAULT = path.join(__dirname, '..', 'orbitdb');
export { API_PORT, WEB3_PROVIDER_URL, RENDEZVOUS_URL };

106
packages/concordia-pinner/src/index.js
View File

@ -1,60 +1,80 @@
import Web3 from 'web3'; import Web3 from 'web3';
import Contract from 'web3-eth-contract'; import Contract from 'web3-eth-contract';
import IPFS from 'ipfs'; import IPFS from 'ipfs';
import { forumContract } from 'concordia-contracts'; import { contracts } from 'concordia-contracts';
import { FORUM_CONTRACT } from 'concordia-app/src/constants/contracts/ContractNames';
import { createOrbitInstance, getPeerDatabases, openKVDBs } from './utils/orbitUtils'; import { createOrbitInstance, getPeerDatabases, openKVDBs } from './utils/orbitUtils';
import ipfsOptions from './options/ipfsOptions'; import ipfsOptions from './options/ipfsOptions';
import { WEB3_PROVIDER_URL } from './constants'; import { WEB3_PROVIDER_URL } from './constants';
import { startAPI } from './app'; import startAPI from './app';
process.on('unhandledRejection', error => { process.on('unhandledRejection', (error) => {
// This happens when attempting to initialize without any available Swarm addresses (e.g. Rendezvous) // This happens when attempting to initialize without any available Swarm addresses (e.g. Rendezvous)
if(error.code === 'ERR_NO_VALID_ADDRESSES'){ if (error.code === 'ERR_NO_VALID_ADDRESSES') {
console.error('unhandledRejection', error.message); console.error('unhandledRejection', error.message);
process.exit(1); process.exit(1);
} }
// Don't swallow other errors
console.error(error);
throw error;
}); });
async function main () { const getDeployedContract = async (web3) => {
const forumContract = contracts.find((contract) => contract.contractName === FORUM_CONTRACT);
return web3.eth.net.getId()
.then((networkId) => forumContract.networks[networkId].address)
.then((contractAddress) => {
Contract.setProvider(WEB3_PROVIDER_URL);
const contract = new Contract(forumContract.abi, contractAddress);
return { contract, contractAddress };
});
};
// Open & replicate databases of existing users
const openExistingUsersDatabases = async (contract, orbit) => contract.methods.getUserAddresses().call()
.then((userAddresses) => getPeerDatabases(orbit, userAddresses))
.then((peerDBs) => openKVDBs(orbit, peerDBs));
const handleWeb3LogEvent = (web3, eventJsonInterface, orbit) => (error, result) => {
if (!error) {
const eventObj = web3.eth.abi.decodeLog(
eventJsonInterface.inputs,
result.data,
result.topics.slice(1),
);
const userAddress = eventObj[1];
console.log('User signed up:', userAddress);
getPeerDatabases(orbit, [userAddress])
.then((peerDBs) => openKVDBs(orbit, peerDBs));
}
};
const main = async () => {
console.log('Initializing...'); console.log('Initializing...');
const web3 = new Web3(new Web3.providers.WebsocketProvider(WEB3_PROVIDER_URL)); const web3 = new Web3(new Web3.providers.WebsocketProvider(WEB3_PROVIDER_URL));
const networkId = await web3.eth.net.getId();
getDeployedContract(web3)
const contractAddress = forumContract.networks[networkId].address; .then(({ contract, contractAddress }) => IPFS.create(ipfsOptions)
.then((ipfs) => createOrbitInstance(ipfs, contractAddress))
Contract.setProvider(WEB3_PROVIDER_URL); .then((orbit) => openExistingUsersDatabases(contract, orbit)
const contract = new Contract(forumContract.abi, contractAddress); .then(() => {
// Listen for new users and subscribe to their databases
const ipfs = await IPFS.create(ipfsOptions); const eventJsonInterface = web3.utils._.find(
const orbit = await createOrbitInstance(ipfs, contractAddress); // eslint-disable-next-line no-underscore-dangle
contract._jsonInterface,
// Open & replicate databases of existing users (obj) => obj.name === 'UserSignedUp' && obj.type === 'event',
const userAddresses = await contract.methods.getUserAddresses().call(); );
const peerDBs = await getPeerDatabases(orbit, userAddresses);
await openKVDBs(orbit, peerDBs); web3.eth.subscribe('logs', {
address: contractAddress,
// Listen for new users and subscribe to their databases topics: [eventJsonInterface.signature],
const eventJsonInterface = web3.utils._.find( }, handleWeb3LogEvent(web3, eventJsonInterface, orbit));
contract._jsonInterface,
obj => obj.name === "UserSignedUp" && obj.type === 'event' startAPI(orbit);
); })));
web3.eth.subscribe('logs', { };
address: contractAddress,
topics: [eventJsonInterface.signature]
}, function(error, result){
if (!error) {
const eventObj = web3.eth.abi.decodeLog(
eventJsonInterface.inputs,
result.data,
result.topics.slice(1)
)
const userAddress = eventObj[1];
console.log(`User signed up:`, userAddress);
getPeerDatabases(orbit, [userAddress]).then(peerDBs => openKVDBs(orbit, peerDBs));
}
});
startAPI(orbit);
}
main(); main();

35
packages/concordia-pinner/src/options/ipfsOptions.js
View File

@ -1,21 +1,22 @@
import libp2pBundle from './libp2pBundle' import libp2pBundle from './libp2pBundle';
import { swarmAddresses } from '../constants'; import { swarmAddresses } from '../constants';
export default { export default {
repo: 'ipfs', repo: 'ipfs',
config: { config: {
Addresses: { Profile: 'server',
Swarm: swarmAddresses Addresses: {
}, Swarm: swarmAddresses,
}, },
libp2p: libp2pBundle, },
EXPERIMENTAL: { libp2p: libp2pBundle,
pubsub: true, EXPERIMENTAL: {
}, pubsub: true,
preload: { },
enabled: false, preload: {
}, enabled: false,
init: { },
emptyRepo: true, init: {
}, emptyRepo: true,
} },
};

140
packages/concordia-pinner/src/options/libp2pBundle.js
View File

@ -10,83 +10,81 @@ import { NOISE } from 'libp2p-noise';
import { swarmAddresses } from '../constants'; import { swarmAddresses } from '../constants';
// See also: https://github.com/libp2p/js-libp2p/blob/master/doc/CONFIGURATION.md // See also: https://github.com/libp2p/js-libp2p/blob/master/doc/CONFIGURATION.md
export default (opts) => { export default (opts) => new Libp2p({
return new Libp2p({ peerId: opts.peerId,
peerId: opts.peerId, addresses: {
addresses: { listen: swarmAddresses,
listen: swarmAddresses },
}, connectionManager: {
connectionManager: { minPeers: 25,
minPeers: 25, maxPeers: 100,
maxPeers: 100, pollInterval: 5000,
pollInterval: 5000 },
}, modules: {
modules: { transport: [
transport: [ WebrtcStar,
WebrtcStar ],
], streamMuxer: [
streamMuxer: [ MPLEX,
MPLEX ],
], connEncryption: [
connEncryption: [ NOISE,
NOISE ],
], peerDiscovery: [
peerDiscovery: [ MulticastDNS,
MulticastDNS, Bootstrap,
Bootstrap ],
], dht: KadDHT,
dht: KadDHT, pubsub: Gossipsub,
pubsub: Gossipsub },
}, config: {
config: { transport: {
transport: { [WebrtcStar.prototype[Symbol.toStringTag]]: {
[WebrtcStar.prototype[Symbol.toStringTag]]: { wrtc,
wrtc
}
}, },
peerDiscovery: { },
autoDial: true, peerDiscovery: {
mdns: { autoDial: true,
enabled: true, mdns: {
interval: 10000 enabled: true,
}, interval: 10000,
bootstrap: {
enabled: true,
interval: 30e3,
list: opts.config.Bootstrap
}
}, },
relay: { bootstrap: {
enabled: true, enabled: true,
hop: { interval: 30e3,
enabled: true, list: opts.config.Bootstrap,
active: true
}
}, },
dht: { },
relay: {
enabled: true,
hop: {
enabled: true, enabled: true,
kBucketSize: 20, active: true,
randomWalk: {
enabled: true,
interval: 10e3,
timeout: 2e3
}
}, },
pubsub: { },
dht: {
enabled: true,
kBucketSize: 20,
randomWalk: {
enabled: true, enabled: true,
emitself: true interval: 10e3,
} timeout: 2e3,
},
}, },
metrics: { pubsub: {
enabled: true, enabled: true,
computeThrottleMaxQueueSize: 1000, emitself: true,
computeThrottleTimeout: 2000, },
movingAverageIntervals: [ },
60 * 1000, // 1 minute metrics: {
5 * 60 * 1000, // 5 minutes enabled: true,
15 * 60 * 1000 // 15 minutes computeThrottleMaxQueueSize: 1000,
], computeThrottleTimeout: 2000,
maxOldPeersRetention: 50 movingAverageIntervals: [
} 60 * 1000, // 1 minute
}) 5 * 60 * 1000, // 5 minutes
} 15 * 60 * 1000, // 15 minutes
],
maxOldPeersRetention: 50,
},
});

75
packages/concordia-pinner/src/utils/orbitUtils.js
View File

@ -2,46 +2,45 @@ import OrbitDB from 'orbit-db';
import Identities from 'orbit-db-identity-provider'; import Identities from 'orbit-db-identity-provider';
import { EthereumContractIdentityProvider } from '@ezerous/eth-identity-provider'; import { EthereumContractIdentityProvider } from '@ezerous/eth-identity-provider';
import Web3 from 'web3'; import Web3 from 'web3';
import { ORBIT_DIRECTORY_DEFAULT } from '../constants';
export async function createOrbitInstance(ipfs, contractAddress){
Identities.addIdentityProvider(EthereumContractIdentityProvider);
EthereumContractIdentityProvider.setWeb3(new Web3()); // We need a fully-featured new Web3 for signature verification
EthereumContractIdentityProvider.setContractAddress(contractAddress);
return await OrbitDB.createInstance(ipfs);
}
export async function getPeerDatabases(orbit, userAddresses) {
const peerDBs = [];
for (const userAddress of userAddresses) {
peerDBs.push(await determineKVAddress({ orbit, dbName:'user', userAddress }));
peerDBs.push(await determineKVAddress({ orbit, dbName:'posts', userAddress }));
peerDBs.push(await determineKVAddress({ orbit, dbName:'topics', userAddress }));
}
return peerDBs;
}
export async function openKVDBs(orbit, databases) {
for (const db of databases){
const store = await orbit.keyvalue(db);
store.events.on('replicated', (address) => console.log(`Replicated ${address}`));
console.log(`Opened ${db}`);
}
}
// TODO: share code below with frontend (?) // TODO: share code below with frontend (?)
async function determineDBAddress({ const determineDBAddress = async ({
orbit, dbName, type, identityId, orbit, dbName, type, identityId,
}) { }) => orbit.determineAddress(dbName, type, { accessController: { write: [identityId] } })
const ipfsMultihash = (await orbit.determineAddress(dbName, type, { .then((orbitAddress) => {
accessController: { write: [identityId] }, const ipfsMultihash = orbitAddress.root;
})).root;
return `/orbitdb/${ipfsMultihash}/${dbName}`; return `/orbitdb/${ipfsMultihash}/${dbName}`;
} });
const determineKVAddress = async ({ orbit, dbName, userAddress }) => determineDBAddress({
orbit, dbName, type: 'keyvalue', identityId: userAddress + EthereumContractIdentityProvider.contractAddress,
});
export const createOrbitInstance = async (ipfs, contractAddress) => {
Identities.addIdentityProvider(EthereumContractIdentityProvider);
EthereumContractIdentityProvider.setWeb3(new Web3()); // We need a fully-featured new Web3 for signature verification
EthereumContractIdentityProvider.setContractAddress(contractAddress);
const ORBIT_DIRECTORY = process.env.ORBIT_DIRECTORY || ORBIT_DIRECTORY_DEFAULT;
return OrbitDB.createInstance(ipfs, { directory: ORBIT_DIRECTORY });
};
export const getPeerDatabases = async (orbit, userAddresses) => Promise.all(userAddresses
.flatMap((userAddress) => [
determineKVAddress({ orbit, dbName: 'user', userAddress }),
determineKVAddress({ orbit, dbName: 'posts', userAddress }),
determineKVAddress({ orbit, dbName: 'topics', userAddress }),
]));
async function determineKVAddress({ orbit, dbName, userAddress }) { export const openKVDBs = async (orbit, databases) => {
return determineDBAddress({ databases
orbit, dbName, type: 'keyvalue', identityId: userAddress + EthereumContractIdentityProvider.contractAddress, .forEach((database) => {
orbit
.keyvalue(database)
.then((store) => store.events.on('replicated', (address) => console.log(`Replicated ${address}`)));
console.log(`Opened ${database}`);
}); });
} };

558
yarn.lock
View File

File diff suppressed because it is too large
Write Preview
Loading…
Cancel
Save