Browse Source

refactor: remove awaits, fix lint issues, add env vars for orbitdb path

develop
Apostolos Fanakis 4 years ago
parent
commit
da5311afea
  1. 60
      packages/concordia-pinner/.eslintrc.js
  2. 2
      packages/concordia-pinner/package.json
  3. 50
      packages/concordia-pinner/src/app.js
  4. 8
      packages/concordia-pinner/src/constants.js
  5. 82
      packages/concordia-pinner/src/index.js
  6. 7
      packages/concordia-pinner/src/options/ipfsOptions.js
  7. 48
      packages/concordia-pinner/src/options/libp2pBundle.js
  8. 67
      packages/concordia-pinner/src/utils/orbitUtils.js
  9. 558
      yarn.lock

60
packages/concordia-pinner/.eslintrc.js

@ -0,0 +1,60 @@
module.exports = {
env: {
browser: true,
es6: true,
jest: true,
},
extends: [
'plugin:react/recommended',
'airbnb',
],
globals: {
Atomics: 'readonly',
SharedArrayBuffer: 'readonly',
},
parser: 'babel-eslint',
parserOptions: {
ecmaFeatures: {
jsx: true,
},
ecmaVersion: 2018,
sourceType: 'module',
},
plugins: [
'react',
'react-hooks',
],
rules: {
'react/jsx-props-no-spreading': 'off',
'import/extensions': 'off',
'react/jsx-indent': [
'error',
4,
{
checkAttributes: true,
indentLogicalExpressions: true,
},
],
'react/require-default-props': 'off',
'react/prop-types': 'off',
'react-hooks/rules-of-hooks': 'error',
'react-hooks/exhaustive-deps': 'error',
'max-len': ['warn', { code: 120, tabWidth: 4 }],
'no-unused-vars': 'warn',
'no-console': 'off',
'no-shadow': 'warn',
'no-multi-str': 'warn',
'jsx-a11y/label-has-associated-control': [2, {
labelAttributes: ['label'],
controlComponents: ['Input'],
depth: 3,
}],
},
settings: {
'import/resolver': {
node: {
extensions: ['.js', '.jsx'],
},
},
},
};

2
packages/concordia-pinner/package.json

@ -11,6 +11,8 @@
"license": "MIT",
"dependencies": {
"@ezerous/eth-identity-provider": "~0.1.2",
"concordia-app": "~0.1.0",
"concordia-contracts": "~0.1.0",
"esm": "~3.2.25",
"express": "^4.17.1",
"ipfs": "~0.52.1",

50
packages/concordia-pinner/src/app.js

@ -1,36 +1,25 @@
import express from 'express';
import _ from 'lodash';
import isReachable from 'is-reachable';
import { API_PORT, RENDEZVOUS_URL, WEB3_PROVIDER_URL } from './constants';
const POLLING_INTERVAL = 1000;
let app;
let responseBody = {
ipfs:{id:"", localAddresses:[], peers:[], totalPeers:0, repoStats:{}},
orbit:{identity:{}, databases:[]},
web3:{url:WEB3_PROVIDER_URL, reachable: false},
rendezvous:{url:RENDEZVOUS_URL, reachable: false},
timestamp:0
const responseBody = {
ipfs: {
id: '', localAddresses: [], peers: [], totalPeers: 0, repoStats: {},
},
orbit: { identity: {}, databases: [] },
web3: { url: WEB3_PROVIDER_URL, reachable: false },
rendezvous: { url: RENDEZVOUS_URL, reachable: false },
timestamp: 0,
};
export function startAPI(orbit){
app = express();
app.get('/', async (req, res) => {
res.send(responseBody);
});
app.listen(API_PORT, () => {
console.log(`Pinner API at http://localhost:${API_PORT}!`);
});
setInterval(getStats, POLLING_INTERVAL, orbit);
}
async function getStats(orbit) {
try {
// eslint-disable-next-line no-underscore-dangle
const ipfs = orbit._ipfs;
const {id} = await ipfs.id();
const { id } = await ipfs.id();
const peers = await ipfs.swarm.peers();
const localAddresses = await ipfs.swarm.localAddrs();
const repoStats = await ipfs.stats.repo();
@ -39,7 +28,7 @@ async function getStats(orbit) {
const databases = Object.keys(orbit.stores);
const isWeb3Reachable = await isReachable(WEB3_PROVIDER_URL);
const isRendezvousReachable = await isReachable(RENDEZVOUS_URL);
const timestamp = + new Date();
const timestamp = +new Date();
responseBody.ipfs.id = id;
responseBody.ipfs.peers = uniquePeers;
@ -52,6 +41,21 @@ async function getStats(orbit) {
responseBody.rendezvous.reachable = isRendezvousReachable;
responseBody.timestamp = timestamp;
} catch (err) {
console.error('Error while getting stats:', err)
console.error('Error while getting stats:', err);
}
}
const startAPI = (orbit) => {
const app = express();
app.get('/', async (req, res) => {
res.send(responseBody);
});
app.listen(API_PORT, () => {
console.log(`Pinner API at http://localhost:${API_PORT}!`);
});
setInterval(getStats, POLLING_INTERVAL, orbit);
};
export default startAPI;

8
packages/concordia-pinner/src/constants.js

@ -1,5 +1,7 @@
import breezeOptions, {RENDEZVOUS_URL} from 'concordia-app/src/options/breezeOptions';
import breezeOptions, { RENDEZVOUS_URL } from 'concordia-app/src/options/breezeOptions';
import { WEB3_HOST_DEFAULT, WEB3_PORT_DEFAULT } from 'concordia-app/src/constants/configuration/defaults';
import path from 'path';
const { WEB3_HOST, WEB3_PORT } = process.env;
const API_PORT = process.env.PINNER_API_PORT || 4444;
@ -10,4 +12,6 @@ const WEB3_PROVIDER_URL = (WEB3_HOST !== undefined && WEB3_PORT !== undefined)
export const swarmAddresses = breezeOptions.ipfs.config.Addresses.Swarm;
export {API_PORT, WEB3_PROVIDER_URL, RENDEZVOUS_URL};
export const ORBIT_DIRECTORY_DEFAULT = path.join(__dirname, '..', 'orbitdb');
export { API_PORT, WEB3_PROVIDER_URL, RENDEZVOUS_URL };

82
packages/concordia-pinner/src/index.js

@ -1,60 +1,80 @@
import Web3 from 'web3';
import Contract from 'web3-eth-contract';
import IPFS from 'ipfs';
import { forumContract } from 'concordia-contracts';
import { contracts } from 'concordia-contracts';
import { FORUM_CONTRACT } from 'concordia-app/src/constants/contracts/ContractNames';
import { createOrbitInstance, getPeerDatabases, openKVDBs } from './utils/orbitUtils';
import ipfsOptions from './options/ipfsOptions';
import { WEB3_PROVIDER_URL } from './constants';
import { startAPI } from './app';
import startAPI from './app';
process.on('unhandledRejection', error => {
process.on('unhandledRejection', (error) => {
// This happens when attempting to initialize without any available Swarm addresses (e.g. Rendezvous)
if(error.code === 'ERR_NO_VALID_ADDRESSES'){
if (error.code === 'ERR_NO_VALID_ADDRESSES') {
console.error('unhandledRejection', error.message);
process.exit(1);
}
});
async function main () {
console.log('Initializing...');
const web3 = new Web3(new Web3.providers.WebsocketProvider(WEB3_PROVIDER_URL));
const networkId = await web3.eth.net.getId();
// Don't swallow other errors
console.error(error);
throw error;
});
const contractAddress = forumContract.networks[networkId].address;
const getDeployedContract = async (web3) => {
const forumContract = contracts.find((contract) => contract.contractName === FORUM_CONTRACT);
return web3.eth.net.getId()
.then((networkId) => forumContract.networks[networkId].address)
.then((contractAddress) => {
Contract.setProvider(WEB3_PROVIDER_URL);
const contract = new Contract(forumContract.abi, contractAddress);
const ipfs = await IPFS.create(ipfsOptions);
const orbit = await createOrbitInstance(ipfs, contractAddress);
return { contract, contractAddress };
});
};
// Open & replicate databases of existing users
const userAddresses = await contract.methods.getUserAddresses().call();
const peerDBs = await getPeerDatabases(orbit, userAddresses);
await openKVDBs(orbit, peerDBs);
// Open & replicate databases of existing users
const openExistingUsersDatabases = async (contract, orbit) => contract.methods.getUserAddresses().call()
.then((userAddresses) => getPeerDatabases(orbit, userAddresses))
.then((peerDBs) => openKVDBs(orbit, peerDBs));
// Listen for new users and subscribe to their databases
const eventJsonInterface = web3.utils._.find(
contract._jsonInterface,
obj => obj.name === "UserSignedUp" && obj.type === 'event'
);
web3.eth.subscribe('logs', {
address: contractAddress,
topics: [eventJsonInterface.signature]
}, function(error, result){
const handleWeb3LogEvent = (web3, eventJsonInterface, orbit) => (error, result) => {
if (!error) {
const eventObj = web3.eth.abi.decodeLog(
eventJsonInterface.inputs,
result.data,
result.topics.slice(1)
)
result.topics.slice(1),
);
const userAddress = eventObj[1];
console.log(`User signed up:`, userAddress);
getPeerDatabases(orbit, [userAddress]).then(peerDBs => openKVDBs(orbit, peerDBs));
console.log('User signed up:', userAddress);
getPeerDatabases(orbit, [userAddress])
.then((peerDBs) => openKVDBs(orbit, peerDBs));
}
});
};
const main = async () => {
console.log('Initializing...');
const web3 = new Web3(new Web3.providers.WebsocketProvider(WEB3_PROVIDER_URL));
getDeployedContract(web3)
.then(({ contract, contractAddress }) => IPFS.create(ipfsOptions)
.then((ipfs) => createOrbitInstance(ipfs, contractAddress))
.then((orbit) => openExistingUsersDatabases(contract, orbit)
.then(() => {
// Listen for new users and subscribe to their databases
const eventJsonInterface = web3.utils._.find(
// eslint-disable-next-line no-underscore-dangle
contract._jsonInterface,
(obj) => obj.name === 'UserSignedUp' && obj.type === 'event',
);
web3.eth.subscribe('logs', {
address: contractAddress,
topics: [eventJsonInterface.signature],
}, handleWeb3LogEvent(web3, eventJsonInterface, orbit));
startAPI(orbit);
}
})));
};
main();

7
packages/concordia-pinner/src/options/ipfsOptions.js

@ -1,11 +1,12 @@
import libp2pBundle from './libp2pBundle'
import libp2pBundle from './libp2pBundle';
import { swarmAddresses } from '../constants';
export default {
repo: 'ipfs',
config: {
Profile: 'server',
Addresses: {
Swarm: swarmAddresses
Swarm: swarmAddresses,
},
},
libp2p: libp2pBundle,
@ -18,4 +19,4 @@ export default {
init: {
emptyRepo: true,
},
}
};

48
packages/concordia-pinner/src/options/libp2pBundle.js

@ -10,58 +10,57 @@ import { NOISE } from 'libp2p-noise';
import { swarmAddresses } from '../constants';
// See also: https://github.com/libp2p/js-libp2p/blob/master/doc/CONFIGURATION.md
export default (opts) => {
return new Libp2p({
export default (opts) => new Libp2p({
peerId: opts.peerId,
addresses: {
listen: swarmAddresses
listen: swarmAddresses,
},
connectionManager: {
minPeers: 25,
maxPeers: 100,
pollInterval: 5000
pollInterval: 5000,
},
modules: {
transport: [
WebrtcStar
WebrtcStar,
],
streamMuxer: [
MPLEX
MPLEX,
],
connEncryption: [
NOISE
NOISE,
],
peerDiscovery: [
MulticastDNS,
Bootstrap
Bootstrap,
],
dht: KadDHT,
pubsub: Gossipsub
pubsub: Gossipsub,
},
config: {
transport: {
[WebrtcStar.prototype[Symbol.toStringTag]]: {
wrtc
}
wrtc,
},
},
peerDiscovery: {
autoDial: true,
mdns: {
enabled: true,
interval: 10000
interval: 10000,
},
bootstrap: {
enabled: true,
interval: 30e3,
list: opts.config.Bootstrap
}
list: opts.config.Bootstrap,
},
},
relay: {
enabled: true,
hop: {
enabled: true,
active: true
}
active: true,
},
},
dht: {
enabled: true,
@ -69,13 +68,13 @@ export default (opts) => {
randomWalk: {
enabled: true,
interval: 10e3,
timeout: 2e3
}
timeout: 2e3,
},
},
pubsub: {
enabled: true,
emitself: true
}
emitself: true,
},
},
metrics: {
enabled: true,
@ -84,9 +83,8 @@ export default (opts) => {
movingAverageIntervals: [
60 * 1000, // 1 minute
5 * 60 * 1000, // 5 minutes
15 * 60 * 1000 // 15 minutes
15 * 60 * 1000, // 15 minutes
],
maxOldPeersRetention: 50
}
})
}
maxOldPeersRetention: 50,
},
});

67
packages/concordia-pinner/src/utils/orbitUtils.js

@ -2,46 +2,45 @@ import OrbitDB from 'orbit-db';
import Identities from 'orbit-db-identity-provider';
import { EthereumContractIdentityProvider } from '@ezerous/eth-identity-provider';
import Web3 from 'web3';
import { ORBIT_DIRECTORY_DEFAULT } from '../constants';
export async function createOrbitInstance(ipfs, contractAddress){
// TODO: share code below with frontend (?)
const determineDBAddress = async ({
orbit, dbName, type, identityId,
}) => orbit.determineAddress(dbName, type, { accessController: { write: [identityId] } })
.then((orbitAddress) => {
const ipfsMultihash = orbitAddress.root;
return `/orbitdb/${ipfsMultihash}/${dbName}`;
});
const determineKVAddress = async ({ orbit, dbName, userAddress }) => determineDBAddress({
orbit, dbName, type: 'keyvalue', identityId: userAddress + EthereumContractIdentityProvider.contractAddress,
});
export const createOrbitInstance = async (ipfs, contractAddress) => {
Identities.addIdentityProvider(EthereumContractIdentityProvider);
EthereumContractIdentityProvider.setWeb3(new Web3()); // We need a fully-featured new Web3 for signature verification
EthereumContractIdentityProvider.setContractAddress(contractAddress);
return await OrbitDB.createInstance(ipfs);
}
export async function getPeerDatabases(orbit, userAddresses) {
const peerDBs = [];
for (const userAddress of userAddresses) {
peerDBs.push(await determineKVAddress({ orbit, dbName:'user', userAddress }));
peerDBs.push(await determineKVAddress({ orbit, dbName:'posts', userAddress }));
peerDBs.push(await determineKVAddress({ orbit, dbName:'topics', userAddress }));
}
return peerDBs;
}
export async function openKVDBs(orbit, databases) {
for (const db of databases){
const store = await orbit.keyvalue(db);
store.events.on('replicated', (address) => console.log(`Replicated ${address}`));
console.log(`Opened ${db}`);
}
}
const ORBIT_DIRECTORY = process.env.ORBIT_DIRECTORY || ORBIT_DIRECTORY_DEFAULT;
// TODO: share code below with frontend (?)
async function determineDBAddress({
orbit, dbName, type, identityId,
}) {
const ipfsMultihash = (await orbit.determineAddress(dbName, type, {
accessController: { write: [identityId] },
})).root;
return `/orbitdb/${ipfsMultihash}/${dbName}`;
}
return OrbitDB.createInstance(ipfs, { directory: ORBIT_DIRECTORY });
};
async function determineKVAddress({ orbit, dbName, userAddress }) {
return determineDBAddress({
orbit, dbName, type: 'keyvalue', identityId: userAddress + EthereumContractIdentityProvider.contractAddress,
export const getPeerDatabases = async (orbit, userAddresses) => Promise.all(userAddresses
.flatMap((userAddress) => [
determineKVAddress({ orbit, dbName: 'user', userAddress }),
determineKVAddress({ orbit, dbName: 'posts', userAddress }),
determineKVAddress({ orbit, dbName: 'topics', userAddress }),
]));
export const openKVDBs = async (orbit, databases) => {
databases
.forEach((database) => {
orbit
.keyvalue(database)
.then((store) => store.events.on('replicated', (address) => console.log(`Replicated ${address}`)));
console.log(`Opened ${database}`);
});
}
};

558
yarn.lock

File diff suppressed because it is too large
Loading…
Cancel
Save