From f67444a5602207c70187712d246591eed0ac29ec Mon Sep 17 00:00:00 2001 From: apostolof Date: Tue, 2 Feb 2021 22:42:49 +0200 Subject: [PATCH 01/18] test: add naive contract tests --- packages/concordia-contracts/test/TestForum.sol | 11 +++++++++++ packages/concordia-contracts/test/forum.js | 11 +++++++++++ 2 files changed, 22 insertions(+) create mode 100755 packages/concordia-contracts/test/TestForum.sol create mode 100755 packages/concordia-contracts/test/forum.js diff --git a/packages/concordia-contracts/test/TestForum.sol b/packages/concordia-contracts/test/TestForum.sol new file mode 100755 index 0000000..c07f277 --- /dev/null +++ b/packages/concordia-contracts/test/TestForum.sol @@ -0,0 +1,11 @@ +pragma solidity >=0.4.21 <0.7.0; + +import "truffle/Assert.sol"; +import "../contracts/Forum.sol"; + +contract TestForum { + function testTrue() public { + + Assert.isTrue(true, "Oops placeholder test failed!"); + } +} \ No newline at end of file diff --git a/packages/concordia-contracts/test/forum.js b/packages/concordia-contracts/test/forum.js new file mode 100755 index 0000000..7120f66 --- /dev/null +++ b/packages/concordia-contracts/test/forum.js @@ -0,0 +1,11 @@ +const Forum = artifacts.require('Forum'); + +contract('Forum', (accounts) => { + it('...should succeed.', async () => { + const forumInstance = await Forum.deployed(); + + assert + .ok(forumInstance + .signUp('testUsername', { from: accounts[0] }), 'The user was not created.'); + }); +}); From ba2d6810f0b63a623b0eeda0b0ba32b526c8498b Mon Sep 17 00:00:00 2001 From: apostolof Date: Tue, 2 Feb 2021 22:44:30 +0200 Subject: [PATCH 02/18] refactor: add staging runtime for app image --- docker/concordia-app/Dockerfile | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) mode change 100644 => 100755 docker/concordia-app/Dockerfile diff --git a/docker/concordia-app/Dockerfile b/docker/concordia-app/Dockerfile old mode 100644 new mode 100755 index 564f0be..0191145 --- a/docker/concordia-app/Dockerfile +++ b/docker/concordia-app/Dockerfile @@ -22,22 +22,24 @@ RUN yarn install --frozen-lockfile COPY ./packages/concordia-contracts ./packages/concordia-contracts COPY ./packages/concordia-app ./packages/concordia-app +# Fix timezome +ARG TZ +ENV TZ=${TZ} +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + # -------------------------------------------------- # Stage 2 (Test) # -------------------------------------------------- FROM base as test -# Fix timezome (needed for timestamps on report files) -ARG TZ -ENV TZ=${TZ} -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone - WORKDIR /opt/concordia-app COPY ./docker/concordia-app/test-app.sh . WORKDIR /usr/src/concordia/packages/concordia-app +RUN chmod +x /opt/concordia-app/test-app.sh + ENTRYPOINT ["/opt/concordia-app/test-app.sh"] # -------------------------------------------------- @@ -50,9 +52,18 @@ WORKDIR /usr/src/concordia/packages/concordia-app RUN yarn build # -------------------------------------------------- -# Stage 4 (Runtime) +# Stage 4 (Staging runtime) +# -------------------------------------------------- +FROM base as staging + +WORKDIR /usr/src/concordia/packages/concordia-app + +ENTRYPOINT ["yarn", "start"] + +# -------------------------------------------------- +# Stage 4 (Production runtime) # -------------------------------------------------- -FROM nginx:1.17-alpine as runtime +FROM nginx:1.17-alpine as production LABEL maintainers.1="Apostolos Fanakis " LABEL maintainers.2="Panagiotis Nikolaidis Date: Tue, 2 Feb 2021 22:45:00 +0200 Subject: [PATCH 03/18] fix: add executable mode to contracts test script --- docker/concordia-contracts/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) mode change 100644 => 100755 docker/concordia-contracts/Dockerfile diff --git a/docker/concordia-contracts/Dockerfile b/docker/concordia-contracts/Dockerfile old mode 100644 new mode 100755 index 414c2b3..7fceb30 --- a/docker/concordia-contracts/Dockerfile +++ b/docker/concordia-contracts/Dockerfile @@ -48,6 +48,8 @@ COPY ./docker/concordia-contracts/test-contracts.sh . WORKDIR /usr/src/concordia/packages/concordia-contracts +RUN ["chmod", "+x", "/opt/concordia-contracts/test-contracts.sh"] + ENTRYPOINT ["/opt/concordia-contracts/test-contracts.sh"] # -------------------------------------------------- From 1e3acd78ac32c68d796a58dbb4be164abe726213 Mon Sep 17 00:00:00 2001 From: apostolof Date: Tue, 2 Feb 2021 22:47:18 +0200 Subject: [PATCH 04/18] refactor: simplify env files, add makefile targets for app staging runtime, include ecentrics in image names, use docker build-kit --- .gitignore | 4 -- docker/Makefile | 42 ++++++++++++------- docker/docker-compose.yml | 4 +- docker/env/concordia.env | 3 ++ docker/env/concordia.example.env | 20 --------- .../{contracts.docker.env => contracts.env} | 7 ---- docker/env/contracts.example.env | 7 ---- .../env/{ganache.docker.env => ganache.env} | 0 ...nache.test.docker.env => ganache.test.env} | 0 9 files changed, 31 insertions(+), 56 deletions(-) mode change 100644 => 100755 .gitignore mode change 100644 => 100755 docker/Makefile mode change 100644 => 100755 docker/docker-compose.yml create mode 100755 docker/env/concordia.env delete mode 100644 docker/env/concordia.example.env rename docker/env/{contracts.docker.env => contracts.env} (51%) mode change 100644 => 100755 delete mode 100644 docker/env/contracts.example.env rename docker/env/{ganache.docker.env => ganache.env} (100%) mode change 100644 => 100755 rename docker/env/{ganache.test.docker.env => ganache.test.env} (100%) mode change 100644 => 100755 diff --git a/.gitignore b/.gitignore old mode 100644 new mode 100755 index 31c619d..9a3b513 --- a/.gitignore +++ b/.gitignore @@ -24,10 +24,6 @@ docker/volumes docker/ganache/volumes docker/reports -# Env var files -docker/env/concordia.env -docker/env/contracts.env - # Misc .env.local .env.development.local diff --git a/docker/Makefile b/docker/Makefile old mode 100644 new mode 100755 index 0b47f47..2a746a5 --- a/docker/Makefile +++ b/docker/Makefile @@ -1,9 +1,13 @@ +.EXPORT_ALL_VARIABLES: PACKAGES := $(abspath ${CURDIR}/../packages) REPORTS := $(abspath ${CURDIR}/reports) GANACHE_VOLUMES := $(abspath ${CURDIR}/ganache/volumes) +DOCKER_BUILDKIT = 1 run: compose-run build-contracts-migrate run-contracts-migrate build-app run-app @echo "Concordia is up and running, head over to http://localhost:7777." +run-staging: compose-run build-contracts-migrate run-contracts-migrate build-app-staging run-app-staging + @echo "Concordia is up and running, head over to http://localhost:7000." # Targets for building/running/stopping the blockchain and rendezvous server (using the docker-compose file) compose-build: @@ -17,13 +21,13 @@ compose-stop-clean-data: # Ganache targets build-ganache: - @docker build ../ -f ./ganache/Dockerfile -t concordia-ganache + @docker build ../ -f ./ganache/Dockerfile -t ecentrics/concordia-ganache run-ganache: @docker network create --driver bridge concordia_ganache_network || true &&\ - docker run -d -v ${GANACHE_VOLUMES}/ganache_keys:/home/ganache_keys -p 8545:8545 --env-file=./env/ganache.docker.env --name concordia-ganache --net=concordia_ganache_network concordia-ganache:latest + docker run -d -v ${GANACHE_VOLUMES}/ganache_keys:/home/ganache_keys -p 8545:8545 --env-file=./env/ganache.env --name concordia-ganache --net=concordia_ganache_network ecentrics/concordia-ganache:latest run-ganache-test: @docker network create --driver bridge concordia_ganache_test_network || true &&\ - docker run --rm -d -p 8546:8546 --env-file=./env/ganache.test.docker.env --name concordia-ganache-test --net=concordia_ganache_test_network concordia-ganache:latest + docker run --rm -d -p 8546:8546 --env-file=./env/ganache.test.env --name concordia-ganache-test --net=concordia_ganache_test_network ecentrics/concordia-ganache:latest # Rendezvous targets run-rendezvous: @@ -32,35 +36,41 @@ run-rendezvous: # Contracts targets build-contracts: - @docker build ../ -f ./concordia-contracts/Dockerfile --target compile -t concordia-contracts --build-arg TZ=Europe/Athens + @docker build ../ -f ./concordia-contracts/Dockerfile --target compile -t ecentrics/concordia-contracts --build-arg TZ=Europe/Athens build-contracts-migrate: - @docker build ../ -f ./concordia-contracts/Dockerfile -t concordia-contracts-migrate --build-arg TZ=Europe/Athens + @docker build ../ -f ./concordia-contracts/Dockerfile -t ecentrics/concordia-contracts-migrate --build-arg TZ=Europe/Athens build-contracts-tests: - @docker build ../ -f ./concordia-contracts/Dockerfile --target test -t concordia-contracts-tests --build-arg TZ=Europe/Athens + @docker build ../ -f ./concordia-contracts/Dockerfile --target test -t ecentrics/concordia-contracts-tests --build-arg TZ=Europe/Athens run-contracts-tests: - @docker run --rm -v ${REPORTS}/contracts/:/usr/test-reports/ --env-file=./env/contracts.docker.env --net=concordia_ganache_test_network concordia-contracts-tests:latest + @docker run --rm -v ${REPORTS}/contracts/:/usr/test-reports/ --env-file=./env/contracts.env --net=concordia_ganache_test_network ecentrics/concordia-contracts-tests:latest run-contracts-tests-host-chain: - @docker run --rm -v ${REPORTS}/contracts/:/usr/test-reports/ --env-file=./env/contracts.env --net=host concordia-contracts-tests:latest + @docker run --rm -v ${REPORTS}/contracts/:/usr/test-reports/ --env-file=./env/contracts.env --net=host ecentrics/concordia-contracts-tests:latest run-contracts-migrate: - @docker run --rm -v ${PACKAGES}/concordia-contracts/build/:/usr/src/concordia/packages/concordia-contracts/build/ --env-file=./env/contracts.docker.env --net=concordia_ganache_network concordia-contracts-migrate:latest + @docker run --rm -v ${PACKAGES}/concordia-contracts/build/:/usr/src/concordia/packages/concordia-contracts/build/ --env-file=./env/contracts.env --net=concordia_ganache_network ecentrics/concordia-contracts-migrate:latest run-contracts-migrate-host-chain: - @docker run --rm -v ${PACKAGES}/concordia-contracts/build/:/usr/src/concordia/packages/concordia-contracts/build/ --env-file=./env/contracts.env --net=host concordia-contracts-migrate:latest + @docker run --rm -v ${PACKAGES}/concordia-contracts/build/:/usr/src/concordia/packages/concordia-contracts/build/ --env-file=./env/contracts.env --net=host ecentrics/concordia-contracts-migrate:latest get-contracts: - @docker run --rm -v ${PACKAGES}/concordia-contracts/build/:/build --entrypoint=sh concordia-contracts:latest -c 'cp /usr/src/concordia/packages/concordia-contracts/build/* /build' + @docker run --rm -v ${PACKAGES}/concordia-contracts/build/:/build --entrypoint=sh ecentrics/concordia-contracts:latest -c 'cp /usr/src/concordia/packages/concordia-contracts/build/* /build' # App targets build-app: - @docker build ../ -f ./concordia-app/Dockerfile -t concordia-app --build-arg TZ=Europe/Athens + @docker build ../ -f ./concordia-app/Dockerfile -t ecentrics/concordia-app --build-arg TZ=Europe/Athens +build-app-staging: + @docker build ../ -f ./concordia-app/Dockerfile --target staging -t ecentrics/concordia-app-staging --build-arg TZ=Europe/Athens build-app-tests: - @docker build ../ -f ./concordia-app/Dockerfile --target test -t concordia-app-tests --build-arg TZ=Europe/Athens + @docker build ../ -f ./concordia-app/Dockerfile --target test -t ecentrics/concordia-app-tests --build-arg TZ=Europe/Athens run-app-tests: - @docker run --rm -v ${REPORTS}/app/:/usr/test-reports/ --env-file=./env/concordia.docker.env concordia-app-tests:latest + @docker run --rm -v ${REPORTS}/app/:/usr/test-reports/ --env-file=./env/concordia.env ecentrics/concordia-app-tests:latest run-app: - @docker create --env-file=./env/concordia.docker.env -p 7777:80 --name concordia-app --net=concordia_ganache_network concordia-app:latest &&\ + @docker create --env-file=./env/concordia.env -p 7777:80 --name concordia-app --net=concordia_ganache_network ecentrics/concordia-app:latest &&\ docker network connect concordia_rendezvous_network concordia-app &&\ docker start concordia-app +run-app-staging: + @docker create -it --env-file=./env/concordia.env -p 7000:3000 --name concordia-app-staging --net=concordia_ganache_network ecentrics/concordia-app-staging:latest &&\ + docker network connect concordia_rendezvous_network concordia-app-staging &&\ + docker start concordia-app-staging run-app-host-chain: - @docker run -d --env-file=./env/concordia.env --name concordia-app --net=host concordia-app:latest + @docker run -d --env-file=./env/concordia.env --name concordia-app --net=host ecentrics/concordia-app:latest # Other clean-images: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml old mode 100644 new mode 100755 index 3fa4386..e01c5c2 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -5,10 +5,10 @@ services: build: context: ../ dockerfile: ./docker/ganache/Dockerfile - image: concordia-ganache + image: ecentrics/concordia-ganache container_name: concordia-ganache env_file: - - env/ganache.docker.env + - env/ganache.env expose: - 8545 ports: diff --git a/docker/env/concordia.env b/docker/env/concordia.env new file mode 100755 index 0000000..d7a24a8 --- /dev/null +++ b/docker/env/concordia.env @@ -0,0 +1,3 @@ +# Variables needed in runtime (in browser) +REACT_APP_RENDEZVOUS_HOST=127.0.0.1 +REACT_APP_RENDEZVOUS_PORT=9090 diff --git a/docker/env/concordia.example.env b/docker/env/concordia.example.env deleted file mode 100644 index c593d66..0000000 --- a/docker/env/concordia.example.env +++ /dev/null @@ -1,20 +0,0 @@ -# Set to "CI" if in CI environment, anything else (including unset) will be ignored -BUILD_ENV={CI} - -# Docker compose variables -VIRTUAL_HOST=example.com -VIRTUAL_PORT=3000 - -# If you uncomment the lines below, Concordia will become available through https BUT the rendezvous -# server will stop working and IPFS initialization won't complete -#LETSENCRYPT_HOST=example.com -#LETSENCRYPT_EMAIL=someemail.email.com - -# Variables needed in runtime -# TO-NEVER-DO: change CONCORDIA_HOST to localhost -CONCORDIA_HOST=0.0.0.0 -CONCORDIA_PORT=3000 - -# Variables needed in runtime (in browser) -REACT_APP_RENDEZVOUS_HOST=xx.xxx.xxx.xxx -REACT_APP_RENDEZVOUS_PORT=9090 diff --git a/docker/env/contracts.docker.env b/docker/env/contracts.env old mode 100644 new mode 100755 similarity index 51% rename from docker/env/contracts.docker.env rename to docker/env/contracts.env index 1b6d49c..de9dc52 --- a/docker/env/contracts.docker.env +++ b/docker/env/contracts.env @@ -5,10 +5,3 @@ DEPLOY_CHAIN_PORT=8545 TEST_CHAIN_HOST=concordia-ganache-test TEST_CHAIN_PORT=8546 - -# If the blockchain is running on host use these instead -#DEPLOY_CHAIN_HOST=127.0.0.1 -#DEPLOY_CHAIN_PORT=8545 - -#TEST_CHAIN_HOST=127.0.0.1 -#TEST_CHAIN_PORT=8546 diff --git a/docker/env/contracts.example.env b/docker/env/contracts.example.env deleted file mode 100644 index 3194b9c..0000000 --- a/docker/env/contracts.example.env +++ /dev/null @@ -1,7 +0,0 @@ -# Variables needed in runtime -MIGRATE_NETWORK=env -DEPLOY_CHAIN_HOST=xx.xxx.xxx.xxx -DEPLOY_CHAIN_PORT=8545 - -TEST_CHAIN_HOST=xx.xxx.xxx.xxx -TEST_CHAIN_PORT=8545 diff --git a/docker/env/ganache.docker.env b/docker/env/ganache.env old mode 100644 new mode 100755 similarity index 100% rename from docker/env/ganache.docker.env rename to docker/env/ganache.env diff --git a/docker/env/ganache.test.docker.env b/docker/env/ganache.test.env old mode 100644 new mode 100755 similarity index 100% rename from docker/env/ganache.test.docker.env rename to docker/env/ganache.test.env From 256f9f4843a4ff6085413febe66a07efa58f277b Mon Sep 17 00:00:00 2001 From: apostolof Date: Thu, 4 Feb 2021 22:48:51 +0200 Subject: [PATCH 05/18] refactor: modify provider service to Promise-based --- .../concordia-contracts-provider/package.json | 1 + .../src/controllers/download.js | 55 +++++++++++-------- .../src/controllers/upload.js | 29 ++++------ .../src/middleware/upload.js | 2 +- .../src/routes/web.js | 5 +- yarn.lock | 5 ++ 6 files changed, 53 insertions(+), 44 deletions(-) diff --git a/packages/concordia-contracts-provider/package.json b/packages/concordia-contracts-provider/package.json index 7bdc166..9c2234e 100755 --- a/packages/concordia-contracts-provider/package.json +++ b/packages/concordia-contracts-provider/package.json @@ -12,6 +12,7 @@ "cors": "^2.8.5", "esm": "~3.2.25", "express": "^4.17.1", + "express-async-handler": "^1.1.4", "lodash": "^4.17.20", "multer": "^1.4.2", "multiparty": "^4.2.2" diff --git a/packages/concordia-contracts-provider/src/controllers/download.js b/packages/concordia-contracts-provider/src/controllers/download.js index fa77b78..baa8151 100755 --- a/packages/concordia-contracts-provider/src/controllers/download.js +++ b/packages/concordia-contracts-provider/src/controllers/download.js @@ -1,31 +1,38 @@ -import * as fs from 'fs'; +import { promises as fs } from 'fs'; import path from 'path'; import { getStorageLocation, getTagsDirectory } from '../utils/storageUtils'; -const downloadContracts = async (req, res) => { - const { params: { hash: hashOrTag } } = req; - let directoryPath = getStorageLocation(hashOrTag); - - if (!fs.existsSync(directoryPath)) { - const tagsDirectory = getTagsDirectory(); - - if (fs.existsSync(tagsDirectory)) { - const tagFilePath = path.join(tagsDirectory, hashOrTag); - const tagReference = fs.readFileSync(tagFilePath, 'utf-8'); - - directoryPath = getStorageLocation(tagReference); - } - } +const readContractFilesToArray = (contractsDirectoryPath) => fs + .readdir(contractsDirectoryPath) + .then((contractFilenames) => contractFilenames + .map((contractFilename) => fs + .readFile(path.join(`${contractsDirectoryPath}/${contractFilename}`), 'utf-8') + .then((rawContractData) => JSON.parse(rawContractData)))) + .then((contractObjectPromises) => Promise.all([...contractObjectPromises])); - const contracts = []; - - fs.readdirSync(directoryPath).forEach((contractFilename) => { - const rawContractData = fs.readFileSync(path.join(`${directoryPath}/${contractFilename}`), 'utf-8'); - const contractJson = JSON.parse(rawContractData); - contracts.push(contractJson); - }); - - res.send(contracts); +const downloadContracts = async (req, res) => { + const { params: { hash: identifier } } = req; + const hashBasedDirectoryPath = getStorageLocation(identifier); + + return fs.access(hashBasedDirectoryPath) + .then(() => readContractFilesToArray(hashBasedDirectoryPath)) + .catch(() => { + const tagsDirectory = getTagsDirectory(); + + return fs + .access(tagsDirectory) + .then(() => { + const tagFilePath = path.join(tagsDirectory, identifier); + + return fs.readFile(tagFilePath, 'utf-8') + .then((tagReference) => { + const tagBasedDirectoryPath = getStorageLocation(tagReference); + + return readContractFilesToArray(tagBasedDirectoryPath); + }); + }); + }).then((contracts) => res.send(contracts)) + .catch(() => Promise.reject(new Error(`No contracts version or tag found for ${identifier}.`))); }; export default downloadContracts; diff --git a/packages/concordia-contracts-provider/src/controllers/upload.js b/packages/concordia-contracts-provider/src/controllers/upload.js index 346c567..894aa40 100755 --- a/packages/concordia-contracts-provider/src/controllers/upload.js +++ b/packages/concordia-contracts-provider/src/controllers/upload.js @@ -1,5 +1,5 @@ import path from 'path'; -import fs from 'fs'; +import { promises as fs } from 'fs'; import upload from '../middleware/upload'; import { getTagsDirectory } from '../utils/storageUtils'; @@ -7,31 +7,26 @@ const addOrTransferTag = (tag, hash) => { const tagsDirectory = getTagsDirectory(); const tagFilePath = path.join(tagsDirectory, tag); - fs.mkdirSync(tagsDirectory, { recursive: true }); - fs.writeFileSync(tagFilePath, hash); + return fs + .mkdir(tagsDirectory, { recursive: true }) + .then(() => fs.writeFile(tagFilePath, hash, 'utf-8')); }; -const uploadContracts = async (req, res) => { - try { - await upload(req, res); +const uploadContracts = async (req, res) => upload(req, res) + .then(() => { + if (req.files.length <= 0) { + return Promise.reject(new Error('You must select at least 1 file.')); + } const { body: { tag } } = req; const { params: { hash } } = req; if (tag) { - addOrTransferTag(tag, hash); - } - - if (req.files.length <= 0) { - return res.send('You must select at least 1 file.'); + return addOrTransferTag(tag, hash) + .then(() => res.send('Files have been uploaded and tagged.')); } return res.send('Files have been uploaded.'); - } catch (error) { - console.log(error); - - return res.send(`Error when trying upload many files: ${error}`); - } -}; + }); export default uploadContracts; diff --git a/packages/concordia-contracts-provider/src/middleware/upload.js b/packages/concordia-contracts-provider/src/middleware/upload.js index c55af11..67ce0b6 100755 --- a/packages/concordia-contracts-provider/src/middleware/upload.js +++ b/packages/concordia-contracts-provider/src/middleware/upload.js @@ -16,7 +16,7 @@ const storage = multer.diskStorage({ if (match.indexOf(file.mimetype) === -1) { const message = `${file.originalname} is invalid. Only JSON files are accepted.`; - return callback(message, null); + callback(message, null); } const filename = `${file.originalname}`; diff --git a/packages/concordia-contracts-provider/src/routes/web.js b/packages/concordia-contracts-provider/src/routes/web.js index 5bdd4fb..61921f4 100755 --- a/packages/concordia-contracts-provider/src/routes/web.js +++ b/packages/concordia-contracts-provider/src/routes/web.js @@ -1,12 +1,13 @@ import express from 'express'; +import asyncHandler from 'express-async-handler'; import downloadContracts from '../controllers/download'; import uploadContracts from '../controllers/upload'; const router = express.Router(); const routes = (app) => { - router.get('/contracts/:hash', downloadContracts); - router.post('/contracts/:hash', uploadContracts); + router.get('/contracts/:hash', asyncHandler(downloadContracts)); + router.post('/contracts/:hash', asyncHandler(uploadContracts)); return app.use('/', router); }; diff --git a/yarn.lock b/yarn.lock index 5123b75..5fe663c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6638,6 +6638,11 @@ explain-error@^1.0.4: resolved "https://registry.yarnpkg.com/explain-error/-/explain-error-1.0.4.tgz#a793d3ac0cad4c6ab571e9968fbbab6cb2532929" integrity sha1-p5PTrAytTGq1cemWj7urbLJTKSk= +express-async-handler@^1.1.4: + version "1.1.4" + resolved "https://registry.yarnpkg.com/express-async-handler/-/express-async-handler-1.1.4.tgz#225a84908df63b35ae9df94b6f0f1af061266426" + integrity sha512-HdmbVF4V4w1q/iz++RV7bUxIeepTukWewiJGkoCKQMtvPF11MLTa7It9PRc/reysXXZSEyD4Pthchju+IUbMiQ== + express@^4.14.0, express@^4.17.1: version "4.17.1" resolved "https://registry.yarnpkg.com/express/-/express-4.17.1.tgz#4491fc38605cf51f8629d39c2b5d026f98a4c134" From 50b5338fe21f962b7b98f39915cf6bdf9d535b9f Mon Sep 17 00:00:00 2001 From: apostolof Date: Thu, 4 Feb 2021 23:18:47 +0200 Subject: [PATCH 06/18] refactor: use more Promises, delete contracts directory if already exists --- .../src/controllers/download.js | 6 +-- .../src/controllers/upload.js | 44 ++++++++++++------- .../src/middleware/upload.js | 2 - 3 files changed, 31 insertions(+), 21 deletions(-) diff --git a/packages/concordia-contracts-provider/src/controllers/download.js b/packages/concordia-contracts-provider/src/controllers/download.js index baa8151..ce23f2e 100755 --- a/packages/concordia-contracts-provider/src/controllers/download.js +++ b/packages/concordia-contracts-provider/src/controllers/download.js @@ -1,4 +1,4 @@ -import { promises as fs } from 'fs'; +import { promises as fs, constants } from 'fs'; import path from 'path'; import { getStorageLocation, getTagsDirectory } from '../utils/storageUtils'; @@ -14,13 +14,13 @@ const downloadContracts = async (req, res) => { const { params: { hash: identifier } } = req; const hashBasedDirectoryPath = getStorageLocation(identifier); - return fs.access(hashBasedDirectoryPath) + return fs.access(hashBasedDirectoryPath, constants.R_OK) .then(() => readContractFilesToArray(hashBasedDirectoryPath)) .catch(() => { const tagsDirectory = getTagsDirectory(); return fs - .access(tagsDirectory) + .access(tagsDirectory, constants.R_OK) .then(() => { const tagFilePath = path.join(tagsDirectory, identifier); diff --git a/packages/concordia-contracts-provider/src/controllers/upload.js b/packages/concordia-contracts-provider/src/controllers/upload.js index 894aa40..a2adee1 100755 --- a/packages/concordia-contracts-provider/src/controllers/upload.js +++ b/packages/concordia-contracts-provider/src/controllers/upload.js @@ -1,7 +1,18 @@ import path from 'path'; -import { promises as fs } from 'fs'; -import upload from '../middleware/upload'; -import { getTagsDirectory } from '../utils/storageUtils'; +import { constants, promises as fs } from 'fs'; +import uploadFilesUsingMiddleware from '../middleware/upload'; +import { getStorageLocation, getTagsDirectory } from '../utils/storageUtils'; + +const provisionContractsDirectory = (req) => { + const { params: { hash } } = req; + const contractsPath = getStorageLocation(hash); + + return fs + .access(contractsPath, constants.W_OK) + .then(() => fs.rmdir(contractsPath, { recursive: true })) + .catch(() => Promise.resolve()) + .then(() => fs.mkdir(contractsPath, { recursive: true })); +}; const addOrTransferTag = (tag, hash) => { const tagsDirectory = getTagsDirectory(); @@ -12,21 +23,22 @@ const addOrTransferTag = (tag, hash) => { .then(() => fs.writeFile(tagFilePath, hash, 'utf-8')); }; -const uploadContracts = async (req, res) => upload(req, res) - .then(() => { - if (req.files.length <= 0) { - return Promise.reject(new Error('You must select at least 1 file.')); - } +const uploadContracts = async (req, res) => provisionContractsDirectory(req) + .then(() => uploadFilesUsingMiddleware(req, res) + .then(() => { + if (req.files.length <= 0) { + return Promise.reject(new Error('You must select at least 1 file.')); + } - const { body: { tag } } = req; - const { params: { hash } } = req; + const { body: { tag } } = req; + const { params: { hash } } = req; - if (tag) { - return addOrTransferTag(tag, hash) - .then(() => res.send('Files have been uploaded and tagged.')); - } + if (tag) { + return addOrTransferTag(tag, hash) + .then(() => res.send('Files have been uploaded and tagged.')); + } - return res.send('Files have been uploaded.'); - }); + return res.send('Files have been uploaded.'); + })); export default uploadContracts; diff --git a/packages/concordia-contracts-provider/src/middleware/upload.js b/packages/concordia-contracts-provider/src/middleware/upload.js index 67ce0b6..c383d14 100755 --- a/packages/concordia-contracts-provider/src/middleware/upload.js +++ b/packages/concordia-contracts-provider/src/middleware/upload.js @@ -1,5 +1,4 @@ import * as util from 'util'; -import * as fs from 'fs'; import multer from 'multer'; import { getStorageLocation } from '../utils/storageUtils'; @@ -8,7 +7,6 @@ const storage = multer.diskStorage({ const { params: { hash } } = req; const contractsPath = getStorageLocation(hash); - fs.mkdirSync(contractsPath, { recursive: true }); callback(null, contractsPath); }, filename: (req, file, callback) => { From 8ad2f3cb49f3e4ba93de4d850e1ffddd4226ffe8 Mon Sep 17 00:00:00 2001 From: apostolof Date: Thu, 4 Feb 2021 23:27:38 +0200 Subject: [PATCH 07/18] refactor: add helmet for protection --- packages/concordia-contracts-provider/package.json | 1 + packages/concordia-contracts-provider/src/index.js | 2 ++ yarn.lock | 5 +++++ 3 files changed, 8 insertions(+) diff --git a/packages/concordia-contracts-provider/package.json b/packages/concordia-contracts-provider/package.json index 9c2234e..f9eacb7 100755 --- a/packages/concordia-contracts-provider/package.json +++ b/packages/concordia-contracts-provider/package.json @@ -13,6 +13,7 @@ "esm": "~3.2.25", "express": "^4.17.1", "express-async-handler": "^1.1.4", + "helmet": "^4.4.1", "lodash": "^4.17.20", "multer": "^1.4.2", "multiparty": "^4.2.2" diff --git a/packages/concordia-contracts-provider/src/index.js b/packages/concordia-contracts-provider/src/index.js index 0d8ddf4..605d10c 100755 --- a/packages/concordia-contracts-provider/src/index.js +++ b/packages/concordia-contracts-provider/src/index.js @@ -1,5 +1,6 @@ import express from 'express'; import cors from 'cors'; +import helmet from 'helmet'; import initRoutes from './routes/web'; import constants from './constants'; @@ -17,6 +18,7 @@ const corsOptions = { app.use(express.urlencoded({ extended: true })); app.use(cors(corsOptions)); +app.use(helmet()); initRoutes(app); diff --git a/yarn.lock b/yarn.lock index 5fe663c..c971c61 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7716,6 +7716,11 @@ heap@~0.2.6: resolved "https://registry.yarnpkg.com/heap/-/heap-0.2.6.tgz#087e1f10b046932fc8594dd9e6d378afc9d1e5ac" integrity sha1-CH4fELBGky/IWU3Z5tN4r8nR5aw= +helmet@^4.4.1: + version "4.4.1" + resolved "https://registry.yarnpkg.com/helmet/-/helmet-4.4.1.tgz#a17e1444d81d7a83ddc6e6f9bc6e2055b994efe7" + integrity sha512-G8tp0wUMI7i8wkMk2xLcEvESg5PiCitFMYgGRc/PwULB0RVhTP5GFdxOwvJwp9XVha8CuS8mnhmE8I/8dx/pbw== + hex-color-regex@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/hex-color-regex/-/hex-color-regex-1.1.0.tgz#4c06fccb4602fe2602b3c93df82d7e7dbf1a8a8e" From bfed2e5112f8f746c03a56485229a2dbb4c59b8e Mon Sep 17 00:00:00 2001 From: apostolof Date: Fri, 5 Feb 2021 00:08:38 +0200 Subject: [PATCH 08/18] feat: add contracts provider Dockerfile and Makefile targets --- .gitignore | 1 + docker/Makefile | 9 +++++ .../concordia-contracts-provider/Dockerfile | 34 +++++++++++++++++++ docker/env/contracts-provider.env | 1 + 4 files changed, 45 insertions(+) create mode 100644 docker/concordia-contracts-provider/Dockerfile create mode 100644 docker/env/contracts-provider.env diff --git a/.gitignore b/.gitignore index 9a3b513..dcddbe1 100755 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ yarn-error.log* # Docker volumes docker/volumes docker/ganache/volumes +docker/concordia-contracts-provider/volumes docker/reports # Misc diff --git a/docker/Makefile b/docker/Makefile index 2a746a5..1ee3d8f 100755 --- a/docker/Makefile +++ b/docker/Makefile @@ -2,6 +2,7 @@ PACKAGES := $(abspath ${CURDIR}/../packages) REPORTS := $(abspath ${CURDIR}/reports) GANACHE_VOLUMES := $(abspath ${CURDIR}/ganache/volumes) +CONTRACTS_PROVIDER_VOLUMES := $(abspath ${CURDIR}/concordia-contracts-provider/volumes) DOCKER_BUILDKIT = 1 run: compose-run build-contracts-migrate run-contracts-migrate build-app run-app @@ -72,6 +73,14 @@ run-app-staging: run-app-host-chain: @docker run -d --env-file=./env/concordia.env --name concordia-app --net=host ecentrics/concordia-app:latest +# Contracts provider targets +build-contracts-provider: + @docker build ../ -f ./concordia-contracts-provider/Dockerfile -t ecentrics/concordia-contracts-provider --build-arg TZ=Europe/Athens +run-contracts-provider-staging: + @docker run -d -v ${CONTRACTS_PROVIDER_VOLUMES}/contracts:/data/contracts --env-file=./env/contracts-provider.env -e NODE_ENV=production -p 8400:8400 --name concordia-contracts-provider ecentrics/concordia-contracts-provider:latest +run-contracts-provider: + @docker run -d -v ${CONTRACTS_PROVIDER_VOLUMES}/contracts:/data/contracts --env-file=./env/contracts-provider.env -p 8400:8400 --name concordia-contracts-provider ecentrics/concordia-contracts-provider:latest + # Other clean-images: @docker rmi `docker images -q -f "dangling=true"` diff --git a/docker/concordia-contracts-provider/Dockerfile b/docker/concordia-contracts-provider/Dockerfile new file mode 100644 index 0000000..452a69d --- /dev/null +++ b/docker/concordia-contracts-provider/Dockerfile @@ -0,0 +1,34 @@ +# -------------------------------------------------- +# Stage 1 (Runtime) +# -------------------------------------------------- +FROM node:14-alpine as runtime +LABEL maintainers.1="Apostolos Fanakis " +LABEL maintainers.2="Panagiotis Nikolaidis " +LABEL gr.thmmy.ecentrics.concordia-image.name="contracts-provider" + +# Fix timezome (needed for timestamps on report files) +ARG TZ + +RUN apk add -U tzdata \ + && cp /usr/share/zoneinfo/$TZ /etc/localtime \ + && echo $TZ > /etc/timezone \ + && apk del tzdata \ + && rm -rf /var/cache/apk/* + +WORKDIR /usr/src/concordia + +# Copy the root package.json and yarn.lock +COPY ./package.json . +COPY ./yarn.lock . + +# Copy the contracts provider package.json, then install modules +COPY ./packages/concordia-contracts-provider/package.json ./packages/concordia-contracts-provider/ + +RUN yarn install --frozen-lockfile --network-timeout 100000 + +# Gets the rest of the source code +COPY ./packages/concordia-contracts-provider ./packages/concordia-contracts-provider + +WORKDIR /usr/src/concordia/packages/concordia-contracts-provider + +ENTRYPOINT ["yarn", "start"] \ No newline at end of file diff --git a/docker/env/contracts-provider.env b/docker/env/contracts-provider.env new file mode 100644 index 0000000..6903a65 --- /dev/null +++ b/docker/env/contracts-provider.env @@ -0,0 +1 @@ +UPLOAD_CONTRACTS_DIRECTORY=/data/contracts/ From 861fdc65e6c2cd552feb7d13fe328f9c0f3aabf8 Mon Sep 17 00:00:00 2001 From: apostolof Date: Sun, 7 Feb 2021 22:19:02 +0200 Subject: [PATCH 09/18] refactor: resolve concordia rendezvous domain --- packages/concordia-pinner/src/app.js | 16 ++++--- packages/concordia-pinner/src/constants.js | 5 +-- packages/concordia-pinner/src/index.js | 5 ++- .../src/options/ipfsOptions.js | 43 ++++++++++--------- .../src/options/libp2pBundle.js | 5 ++- .../concordia-pinner/src/utils/ipfsUtils.js | 11 +++++ .../concordia-pinner/src/utils/orbitUtils.js | 2 +- 7 files changed, 54 insertions(+), 33 deletions(-) create mode 100644 packages/concordia-pinner/src/utils/ipfsUtils.js diff --git a/packages/concordia-pinner/src/app.js b/packages/concordia-pinner/src/app.js index 690e450..32e628e 100644 --- a/packages/concordia-pinner/src/app.js +++ b/packages/concordia-pinner/src/app.js @@ -3,7 +3,7 @@ import _ from 'lodash'; import isReachable from 'is-reachable'; import { pinnerApiPort } from 'concordia-shared/src/environment/interpolated/pinner'; import getWeb3ProviderUrl from 'concordia-shared/src/utils/web3'; -import getRendezvousUrl from 'concordia-shared/src/utils/rendezvous'; +import { getResolvedRendezvousUrl } from './utils/ipfsUtils'; const POLLING_INTERVAL = 1000; @@ -13,11 +13,17 @@ const responseBody = { }, orbit: { identity: {}, databases: [] }, web3: { url: getWeb3ProviderUrl(), reachable: false }, - rendezvous: { url: getRendezvousUrl(), reachable: false }, + rendezvous: { url: '', reachable: false }, timestamp: 0, }; -async function getStats(orbit) { +getResolvedRendezvousUrl().then(({ address }) => { + responseBody.rendezvous.url = address; +}); + +const getStats = async (orbit) => { + const { address: resolvedRendezvousUrl } = await getResolvedRendezvousUrl(); + try { // eslint-disable-next-line no-underscore-dangle const ipfs = orbit._ipfs; @@ -29,7 +35,7 @@ async function getStats(orbit) { const orbitIdentity = orbit.identity; const databases = Object.keys(orbit.stores); const isWeb3Reachable = await isReachable(getWeb3ProviderUrl()); - const isRendezvousReachable = await isReachable(getRendezvousUrl()); + const isRendezvousReachable = await isReachable(resolvedRendezvousUrl.address); const timestamp = +new Date(); responseBody.ipfs.id = id; @@ -45,7 +51,7 @@ async function getStats(orbit) { } catch (err) { console.error('Error while getting stats:', err); } -} +}; const startAPI = (orbit) => { const app = express(); diff --git a/packages/concordia-pinner/src/constants.js b/packages/concordia-pinner/src/constants.js index 6a8055f..bdec2d4 100644 --- a/packages/concordia-pinner/src/constants.js +++ b/packages/concordia-pinner/src/constants.js @@ -1,6 +1,5 @@ import path from 'path'; -import getBreezeConfiguration from 'concordia-shared/src/configuration/breezeConfiguration'; -export const swarmAddresses = getBreezeConfiguration().ipfs.config.Addresses.Swarm; +const ORBIT_DIRECTORY_DEFAULT = path.join(__dirname, '..', 'orbitdb'); -export const ORBIT_DIRECTORY_DEFAULT = path.join(__dirname, '..', 'orbitdb'); +export default ORBIT_DIRECTORY_DEFAULT; diff --git a/packages/concordia-pinner/src/index.js b/packages/concordia-pinner/src/index.js index aa5dbb2..f869266 100644 --- a/packages/concordia-pinner/src/index.js +++ b/packages/concordia-pinner/src/index.js @@ -5,9 +5,9 @@ import { contracts } from 'concordia-contracts'; import { FORUM_CONTRACT } from 'concordia-shared/src/constants/contracts/ContractNames'; import getWeb3ProviderUrl from 'concordia-shared/src/utils/web3'; import { createOrbitInstance, getPeerDatabases, openKVDBs } from './utils/orbitUtils'; -import ipfsOptions from './options/ipfsOptions'; import startAPI from './app'; import downloadContractArtifacts from './utils/drizzleUtils'; +import getIpfsOptions from './options/ipfsOptions'; process.on('unhandledRejection', (error) => { // This happens when attempting to initialize without any available Swarm addresses (e.g. Rendezvous) @@ -68,7 +68,8 @@ const main = async () => { const web3 = new Web3(new Web3.providers.WebsocketProvider(getWeb3ProviderUrl())); getDeployedContract(web3) - .then(({ contract, contractAddress }) => IPFS.create(ipfsOptions) + .then(({ contract, contractAddress }) => getIpfsOptions() + .then((ipfsOptions) => IPFS.create(ipfsOptions)) .then((ipfs) => createOrbitInstance(ipfs, contractAddress)) .then((orbit) => openExistingUsersDatabases(contract, orbit) .then(() => { diff --git a/packages/concordia-pinner/src/options/ipfsOptions.js b/packages/concordia-pinner/src/options/ipfsOptions.js index b25b5e6..3e56cc9 100644 --- a/packages/concordia-pinner/src/options/ipfsOptions.js +++ b/packages/concordia-pinner/src/options/ipfsOptions.js @@ -1,22 +1,25 @@ -import libp2pBundle from './libp2pBundle'; -import { swarmAddresses } from '../constants'; +import getLibp2pBundle from './libp2pBundle'; +import { getSwarmAddresses } from '../utils/ipfsUtils'; -export default { - repo: 'ipfs', - config: { - Profile: 'server', - Addresses: { - Swarm: swarmAddresses, +const getIpfsOptions = async () => getSwarmAddresses() + .then((swarmAddresses) => ({ + repo: 'ipfs', + config: { + Profile: 'server', + Addresses: { + Swarm: swarmAddresses, + }, }, - }, - libp2p: libp2pBundle, - EXPERIMENTAL: { - pubsub: true, - }, - preload: { - enabled: false, - }, - init: { - emptyRepo: true, - }, -}; + libp2p: getLibp2pBundle(swarmAddresses), + EXPERIMENTAL: { + pubsub: true, + }, + preload: { + enabled: false, + }, + init: { + emptyRepo: true, + }, + })); + +export default getIpfsOptions; diff --git a/packages/concordia-pinner/src/options/libp2pBundle.js b/packages/concordia-pinner/src/options/libp2pBundle.js index 4de108b..94c0b08 100644 --- a/packages/concordia-pinner/src/options/libp2pBundle.js +++ b/packages/concordia-pinner/src/options/libp2pBundle.js @@ -7,10 +7,9 @@ import Gossipsub from 'libp2p-gossipsub'; import KadDHT from 'libp2p-kad-dht'; import MPLEX from 'libp2p-mplex'; import { NOISE } from 'libp2p-noise'; -import { swarmAddresses } from '../constants'; // See also: https://github.com/libp2p/js-libp2p/blob/master/doc/CONFIGURATION.md -export default (opts) => new Libp2p({ +const getLibp2pBundle = (swarmAddresses) => (opts) => new Libp2p({ peerId: opts.peerId, addresses: { listen: swarmAddresses, @@ -88,3 +87,5 @@ export default (opts) => new Libp2p({ maxOldPeersRetention: 50, }, }); + +export default getLibp2pBundle; diff --git a/packages/concordia-pinner/src/utils/ipfsUtils.js b/packages/concordia-pinner/src/utils/ipfsUtils.js new file mode 100644 index 0000000..6fa68c2 --- /dev/null +++ b/packages/concordia-pinner/src/utils/ipfsUtils.js @@ -0,0 +1,11 @@ +import dns from 'dns'; +import util from 'util'; +import { rendezvousHost, rendezvousPort } from 'concordia-shared/src/environment/interpolated/rendezvous'; + +const dnsLookup = util.promisify(dns.lookup); + +export const getResolvedRendezvousUrl = async () => dnsLookup(rendezvousHost, { family: 4 }) + .catch((error) => console.error(new Error(`DNS lookup of ${rendezvousHost} failed.\n${error}`))); + +export const getSwarmAddresses = async () => getResolvedRendezvousUrl() + .then((resolvedRendezvousUrl) => [`/ip4/${resolvedRendezvousUrl.address}/tcp/${rendezvousPort}/wss/p2p-webrtc-star`]); diff --git a/packages/concordia-pinner/src/utils/orbitUtils.js b/packages/concordia-pinner/src/utils/orbitUtils.js index 71f47bf..74aae64 100644 --- a/packages/concordia-pinner/src/utils/orbitUtils.js +++ b/packages/concordia-pinner/src/utils/orbitUtils.js @@ -2,7 +2,7 @@ import OrbitDB from 'orbit-db'; import Identities from 'orbit-db-identity-provider'; import { EthereumContractIdentityProvider } from '@ezerous/eth-identity-provider'; import Web3 from 'web3'; -import { ORBIT_DIRECTORY_DEFAULT } from '../constants'; +import ORBIT_DIRECTORY_DEFAULT from '../constants'; // TODO: share code below with frontend (?) const determineDBAddress = async ({ From dcccd03b8036c64b8a4d69a8107e82e5d336ca88 Mon Sep 17 00:00:00 2001 From: apostolof Date: Sun, 7 Feb 2021 22:57:58 +0200 Subject: [PATCH 10/18] fix: minor method call fix --- packages/concordia-app/src/redux/store.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/concordia-app/src/redux/store.js b/packages/concordia-app/src/redux/store.js index 34dedbe..25e0e9e 100644 --- a/packages/concordia-app/src/redux/store.js +++ b/packages/concordia-app/src/redux/store.js @@ -28,9 +28,7 @@ const store = configureStore({ preloadedState: initialState, }); -const breezeConfiguration = getBreezeConfiguration(EthereumContractIdentityProvider, - process.env.REACT_APP_RENDEZVOUS_HOST, - process.env.REACT_APP_RENDEZVOUS_PORT); +const breezeConfiguration = getBreezeConfiguration(EthereumContractIdentityProvider); export const drizzle = new Drizzle(drizzleOptions, store); export const breeze = new Breeze(breezeConfiguration, store); From cd5bd0cd944a3155f9ffe7aff47370ad601745b8 Mon Sep 17 00:00:00 2001 From: apostolof Date: Sun, 7 Feb 2021 22:58:52 +0200 Subject: [PATCH 11/18] fix: copy shared package in docker images --- .gitignore | 1 + docker/Makefile | 38 ++++++++----------- docker/concordia-app/Dockerfile | 6 ++- .../concordia-contracts-provider/Dockerfile | 4 +- docker/concordia-contracts/Dockerfile | 4 +- docker/concordia-contracts/migrate.sh | 2 +- docker/concordia-pinner/Dockerfile | 2 +- docker/docker-compose.yml | 7 ++-- docker/env/concordia.env | 1 + docker/env/contracts.env | 7 +++- docker/env/pinner.env | 7 ++-- 11 files changed, 41 insertions(+), 38 deletions(-) diff --git a/.gitignore b/.gitignore index 9cafa6c..7bbaae4 100755 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,7 @@ yarn-error.log* docker/volumes docker/ganache/volumes docker/concordia-contracts-provider/volumes +docker/concordia-pinner/volumes docker/reports # Misc diff --git a/docker/Makefile b/docker/Makefile index bcefc51..8645f13 100755 --- a/docker/Makefile +++ b/docker/Makefile @@ -4,11 +4,11 @@ REPORTS := $(abspath ${CURDIR}/reports) GANACHE_VOLUMES := $(abspath ${CURDIR}/ganache/volumes) CONTRACTS_PROVIDER_VOLUMES := $(abspath ${CURDIR}/concordia-contracts-provider/volumes) DOCKER_BUILDKIT = 1 -PINNER_VOLUMES := $(abspath ${CURDIR}/pinner/volumes) +PINNER_VOLUMES := $(abspath ${CURDIR}/concordia-pinner/volumes) -run: compose-run build-contracts-migrate run-contracts-migrate build-app run-app +run: compose-run build-contracts-provider run-contracts-provider build-contracts-migrate run-contracts-migrate build-pinner run-pinner build-app run-app @echo "Concordia is up and running, head over to http://localhost:7777." -run-staging: compose-run build-contracts-migrate run-contracts-migrate build-app-staging run-app-staging +run-staging: compose-run build-contracts-provider run-contracts-provider-staging build-contracts-migrate run-contracts-migrate build-pinner run-pinner-staging build-app-staging run-app-staging @echo "Concordia is up and running, head over to http://localhost:7000." # Targets for building/running/stopping the blockchain and rendezvous server (using the docker-compose file) @@ -25,16 +25,14 @@ compose-stop-clean-data: build-ganache: @docker build ../ -f ./ganache/Dockerfile -t ecentrics/concordia-ganache run-ganache: - @docker network create --driver bridge concordia_ganache_network || true &&\ - docker run -d -v ${GANACHE_VOLUMES}/ganache_keys:/home/ganache_keys -p 8545:8545 --env-file=./env/ganache.env --name concordia-ganache --net=concordia_ganache_network ecentrics/concordia-ganache:latest + @docker run -d -v ${GANACHE_VOLUMES}/ganache_keys:/home/ganache_keys -p 8545:8545 --env-file=./env/ganache.env --name concordia-ganache --net=concordia_concordia_network ecentrics/concordia-ganache:latest run-ganache-test: - @docker network create --driver bridge concordia_ganache_test_network || true &&\ - docker run --rm -d -p 8546:8546 --env-file=./env/ganache.test.env --name concordia-ganache-test --net=concordia_ganache_test_network ecentrics/concordia-ganache:latest + @docker run --rm -d -p 8546:8546 --env-file=./env/ganache.test.env --name concordia-ganache-test --net=concordia_concordia_network ecentrics/concordia-ganache:latest # Rendezvous targets run-rendezvous: - @docker network create --driver bridge concordia_rendezvous_network || true &&\ - docker run -d -p 9090:9090 --name concordia-rendezvous libp2p/js-libp2p-webrtc-star:version-0.20.5 + @docker network create --driver bridge concordia_concordia_network || true &&\ + docker run -d -p 9090:9090 --name concordia-rendezvous --net=concordia_concordia_network libp2p/js-libp2p-webrtc-star:version-0.20.5 # Contracts targets build-contracts: @@ -44,11 +42,11 @@ build-contracts-migrate: build-contracts-tests: @docker build ../ -f ./concordia-contracts/Dockerfile --target test -t ecentrics/concordia-contracts-tests --build-arg TZ=Europe/Athens run-contracts-tests: - @docker run --rm -v ${REPORTS}/contracts/:/usr/test-reports/ --env-file=./env/contracts.env --net=concordia_ganache_test_network ecentrics/concordia-contracts-tests:latest + @docker run --rm -v ${REPORTS}/contracts/:/usr/test-reports/ --env-file=./env/contracts.env --net=concordia_concordia_network ecentrics/concordia-contracts-tests:latest run-contracts-tests-host-chain: @docker run --rm -v ${REPORTS}/contracts/:/usr/test-reports/ --env-file=./env/contracts.env --net=host ecentrics/concordia-contracts-tests:latest run-contracts-migrate: - @docker run --rm -v ${PACKAGES}/concordia-contracts/build/:/usr/src/concordia/packages/concordia-contracts/build/ --env-file=./env/contracts.env --net=concordia_ganache_network ecentrics/concordia-contracts-migrate:latest + @docker run --rm -v ${PACKAGES}/concordia-contracts/build/:/usr/src/concordia/packages/concordia-contracts/build/ --env-file=./env/contracts.env --net=concordia_concordia_network ecentrics/concordia-contracts-migrate:latest run-contracts-migrate-host-chain: @docker run --rm -v ${PACKAGES}/concordia-contracts/build/:/usr/src/concordia/packages/concordia-contracts/build/ --env-file=./env/contracts.env --net=host ecentrics/concordia-contracts-migrate:latest get-contracts: @@ -64,13 +62,9 @@ build-app-tests: run-app-tests: @docker run --rm -v ${REPORTS}/app/:/usr/test-reports/ --env-file=./env/concordia.env ecentrics/concordia-app-tests:latest run-app: - @docker create --env-file=./env/concordia.env -p 7777:80 --name concordia-app --net=concordia_ganache_network ecentrics/concordia-app:latest &&\ - docker network connect concordia_rendezvous_network concordia-app &&\ - docker start concordia-app + @docker run -d --env-file=./env/concordia.env -p 7777:80 --name concordia-app ecentrics/concordia-app:latest run-app-staging: - @docker create -it --env-file=./env/concordia.env -p 7000:3000 --name concordia-app-staging --net=concordia_ganache_network ecentrics/concordia-app-staging:latest &&\ - docker network connect concordia_rendezvous_network concordia-app-staging &&\ - docker start concordia-app-staging + @docker run -itd --env-file=./env/concordia.env -p 7000:3000 --name concordia-app-staging ecentrics/concordia-app-staging:latest run-app-host-chain: @docker run -d --env-file=./env/concordia.env --name concordia-app --net=host ecentrics/concordia-app:latest @@ -78,17 +72,17 @@ run-app-host-chain: build-contracts-provider: @docker build ../ -f ./concordia-contracts-provider/Dockerfile -t ecentrics/concordia-contracts-provider --build-arg TZ=Europe/Athens run-contracts-provider-staging: - @docker run -d -v ${CONTRACTS_PROVIDER_VOLUMES}/contracts:/data/contracts --env-file=./env/contracts-provider.env -e NODE_ENV=production -p 8400:8400 --name concordia-contracts-provider ecentrics/concordia-contracts-provider:latest + @docker run -d -v ${CONTRACTS_PROVIDER_VOLUMES}/contracts:/data/contracts --env-file=./env/contracts-provider.env -e NODE_ENV=production -p 8400:8400 --name concordia-contracts-provider --net=concordia_concordia_network ecentrics/concordia-contracts-provider:latest run-contracts-provider: - @docker run -d -v ${CONTRACTS_PROVIDER_VOLUMES}/contracts:/data/contracts --env-file=./env/contracts-provider.env -p 8400:8400 --name concordia-contracts-provider ecentrics/concordia-contracts-provider:latest + @docker run -d -v ${CONTRACTS_PROVIDER_VOLUMES}/contracts:/data/contracts --env-file=./env/contracts-provider.env -p 8400:8400 --name concordia-contracts-provider --net=concordia_concordia_network ecentrics/concordia-contracts-provider:latest -# Contracts provider targets +# Pinner targets build-pinner: @docker build ../ -f ./concordia-pinner/Dockerfile -t ecentrics/concordia-pinner --build-arg TZ=Europe/Athens run-pinner-staging: - @docker run -d -v ${PINNER_VOLUMES}/orbitdb:/data/orbitdb --env-file=./env/pinner.env -e NODE_ENV=production -p 4444:4444 --name concordia-pinner ecentrics/concordia-pinner:latest + @docker run -d -v ${PINNER_VOLUMES}/orbitdb:/data/orbitdb --env-file=./env/pinner.env -e NODE_ENV=production -p 4444:4444 --name concordia-pinner --net=concordia_concordia_network ecentrics/concordia-pinner:latest run-pinner: - @docker run -d -v ${PINNER_VOLUMES}/orbitdb:/data/orbitdb --env-file=./env/pinner.env -p 4444:4444 --name concordia-pinner ecentrics/concordia-pinner:latest + @docker run -d -v ${PINNER_VOLUMES}/orbitdb:/data/orbitdb --env-file=./env/pinner.env -p 4444:4444 --name concordia-pinner --net=concordia_concordia_network ecentrics/concordia-pinner:latest run-pinner-staging-host: @docker run -d -v ${PINNER_VOLUMES}/orbitdb:/data/orbitdb --env-file=./env/pinner.env -e NODE_ENV=production --net=host --name concordia-pinner ecentrics/concordia-pinner:latest run-pinner-host: diff --git a/docker/concordia-app/Dockerfile b/docker/concordia-app/Dockerfile index 0191145..132ec8b 100755 --- a/docker/concordia-app/Dockerfile +++ b/docker/concordia-app/Dockerfile @@ -12,14 +12,16 @@ WORKDIR /usr/src/concordia COPY ./package.json . COPY ./yarn.lock . -# Copy package.json files from contracts and app, then install base modules +# Copy package.json files from contracts, shared and app, then install modules COPY ./packages/concordia-contracts/package.json ./packages/concordia-contracts/package.json +COPY ./packages/concordia-shared/package.json ./packages/concordia-shared/package.json COPY ./packages/concordia-app/package.json ./packages/concordia-app/ RUN yarn install --frozen-lockfile # Gets the rest of the source code COPY ./packages/concordia-contracts ./packages/concordia-contracts +COPY ./packages/concordia-shared ./packages/concordia-shared COPY ./packages/concordia-app ./packages/concordia-app # Fix timezome @@ -38,7 +40,7 @@ COPY ./docker/concordia-app/test-app.sh . WORKDIR /usr/src/concordia/packages/concordia-app -RUN chmod +x /opt/concordia-app/test-app.sh +RUN ["chmod", "+x", "/opt/concordia-app/test-app.sh"] ENTRYPOINT ["/opt/concordia-app/test-app.sh"] diff --git a/docker/concordia-contracts-provider/Dockerfile b/docker/concordia-contracts-provider/Dockerfile index 452a69d..4fb356e 100644 --- a/docker/concordia-contracts-provider/Dockerfile +++ b/docker/concordia-contracts-provider/Dockerfile @@ -21,12 +21,14 @@ WORKDIR /usr/src/concordia COPY ./package.json . COPY ./yarn.lock . -# Copy the contracts provider package.json, then install modules +# Copy package.json files from shared and contracts provider, then install modules +COPY ./packages/concordia-shared/package.json ./packages/concordia-shared/ COPY ./packages/concordia-contracts-provider/package.json ./packages/concordia-contracts-provider/ RUN yarn install --frozen-lockfile --network-timeout 100000 # Gets the rest of the source code +COPY ./packages/concordia-shared ./packages/concordia-shared COPY ./packages/concordia-contracts-provider ./packages/concordia-contracts-provider WORKDIR /usr/src/concordia/packages/concordia-contracts-provider diff --git a/docker/concordia-contracts/Dockerfile b/docker/concordia-contracts/Dockerfile index 7fceb30..fe3b45c 100755 --- a/docker/concordia-contracts/Dockerfile +++ b/docker/concordia-contracts/Dockerfile @@ -21,12 +21,14 @@ WORKDIR /usr/src/concordia COPY ./package.json . COPY ./yarn.lock . -# Copy the contracts package.json, then install modules +# Copy package.json files from shared and contracts, then install modules +COPY ./packages/concordia-shared/package.json ./packages/concordia-shared/ COPY ./packages/concordia-contracts/package.json ./packages/concordia-contracts/ RUN yarn install --frozen-lockfile --network-timeout 100000 # Gets the rest of the source code +COPY ./packages/concordia-shared ./packages/concordia-shared COPY ./packages/concordia-contracts ./packages/concordia-contracts # -------------------------------------------------- diff --git a/docker/concordia-contracts/migrate.sh b/docker/concordia-contracts/migrate.sh index 29d7449..20a813f 100644 --- a/docker/concordia-contracts/migrate.sh +++ b/docker/concordia-contracts/migrate.sh @@ -3,4 +3,4 @@ export CHAIN_HOST="$DEPLOY_CHAIN_HOST" export CHAIN_PORT="$DEPLOY_CHAIN_PORT" -cd /usr/src/concordia/packages/concordia-contracts && yarn _migrate --network "${MIGRATE_NETWORK}" --reset +cd /usr/src/concordia/packages/concordia-contracts && yarn _migrate --network "${MIGRATE_NETWORK}" --reset && yarn upload diff --git a/docker/concordia-pinner/Dockerfile b/docker/concordia-pinner/Dockerfile index e44ad70..3a2bf8b 100644 --- a/docker/concordia-pinner/Dockerfile +++ b/docker/concordia-pinner/Dockerfile @@ -17,7 +17,7 @@ WORKDIR /usr/src/concordia COPY ./package.json . COPY ./yarn.lock . -# Copy the pinner, contracts and shared package.json files, then install modules +# Copy package.json files from contracts, shared and pinner, then install modules COPY ./packages/concordia-pinner/package.json ./packages/concordia-pinner/ COPY ./packages/concordia-contracts/package.json ./packages/concordia-contracts/ COPY ./packages/concordia-shared/package.json ./packages/concordia-shared/ diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index e01c5c2..9ba19a1 100755 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -17,18 +17,17 @@ services: volumes: - ./ganache/volumes/ganache_keys:/home/ganache_keys networks: - ganache_network: + concordia_network: restart: always rendezvous: image: libp2p/js-libp2p-webrtc-star:version-0.20.5 container_name: concordia-rendezvous networks: - rendezvous_network: + concordia_network: ports: - 9090:9090 restart: always networks: - ganache_network: - rendezvous_network: + concordia_network: diff --git a/docker/env/concordia.env b/docker/env/concordia.env index d7a24a8..42652e7 100755 --- a/docker/env/concordia.env +++ b/docker/env/concordia.env @@ -1,3 +1,4 @@ # Variables needed in runtime (in browser) REACT_APP_RENDEZVOUS_HOST=127.0.0.1 REACT_APP_RENDEZVOUS_PORT=9090 +USE_EXTERNAL_CONTRACTS_PROVIDER=true diff --git a/docker/env/contracts.env b/docker/env/contracts.env index de9dc52..f5d51f8 100755 --- a/docker/env/contracts.env +++ b/docker/env/contracts.env @@ -1,7 +1,10 @@ # Variables needed in runtime MIGRATE_NETWORK=env -DEPLOY_CHAIN_HOST=concordia-ganache -DEPLOY_CHAIN_PORT=8545 +WEB3_HOST=concordia-ganache +WEB3_PORT=8545 + +CONTRACTS_PROVIDER_HOST=concordia-contracts-provider +CONTRACTS_PROVIDER_PORT=8400 TEST_CHAIN_HOST=concordia-ganache-test TEST_CHAIN_PORT=8546 diff --git a/docker/env/pinner.env b/docker/env/pinner.env index e723d45..84fb01e 100644 --- a/docker/env/pinner.env +++ b/docker/env/pinner.env @@ -1,16 +1,15 @@ USE_EXTERNAL_CONTRACTS_PROVIDER=true ORBIT_DIRECTORY=/data/orbitdb -#CONTRACTS_PROVIDER_HOST=contracts-provider -CONTRACTS_PROVIDER_HOST=127.0.0.1 +CONTRACTS_PROVIDER_HOST=concordia-contracts-provider CONTRACTS_PROVIDER_PORT=8400 CONTRACTS_VERSION_HASH=latest PINNER_API_HOST=127.0.0.1 PINNER_API_PORT=4444 -RENDEZVOUS_HOST=127.0.0.1 +RENDEZVOUS_HOST=concordia-rendezvous RENDEZVOUS_PORT=9090 -WEB3_HOST=127.0.0.1 +WEB3_HOST=concordia-ganache WEB3_PORT=8545 From 51a365729ac7f652ae7e9e30b2a6613cbab5cfa6 Mon Sep 17 00:00:00 2001 From: apostolof Date: Mon, 8 Feb 2021 21:16:04 +0200 Subject: [PATCH 12/18] refactor: lock docker images, normalize mount paths, minor fixes --- docker/Makefile | 24 +++++++++---------- docker/concordia-app/Dockerfile | 7 ++++-- docker/concordia-app/test-app.sh | 2 +- .../concordia-contracts-provider/Dockerfile | 2 +- docker/concordia-contracts/Dockerfile | 2 +- docker/concordia-contracts/test-contracts.sh | 6 ++--- docker/concordia-pinner/Dockerfile | 2 +- docker/docker-compose.yml | 2 +- docker/env/contracts-provider.env | 2 +- docker/env/pinner.env | 2 +- docker/ganache/Dockerfile | 2 +- docker/ganache/start-blockchain.sh | 8 +++---- 12 files changed, 32 insertions(+), 29 deletions(-) diff --git a/docker/Makefile b/docker/Makefile index 8645f13..3ddded8 100755 --- a/docker/Makefile +++ b/docker/Makefile @@ -3,8 +3,8 @@ PACKAGES := $(abspath ${CURDIR}/../packages) REPORTS := $(abspath ${CURDIR}/reports) GANACHE_VOLUMES := $(abspath ${CURDIR}/ganache/volumes) CONTRACTS_PROVIDER_VOLUMES := $(abspath ${CURDIR}/concordia-contracts-provider/volumes) -DOCKER_BUILDKIT = 1 PINNER_VOLUMES := $(abspath ${CURDIR}/concordia-pinner/volumes) +DOCKER_BUILDKIT = 1 run: compose-run build-contracts-provider run-contracts-provider build-contracts-migrate run-contracts-migrate build-pinner run-pinner build-app run-app @echo "Concordia is up and running, head over to http://localhost:7777." @@ -25,7 +25,7 @@ compose-stop-clean-data: build-ganache: @docker build ../ -f ./ganache/Dockerfile -t ecentrics/concordia-ganache run-ganache: - @docker run -d -v ${GANACHE_VOLUMES}/ganache_keys:/home/ganache_keys -p 8545:8545 --env-file=./env/ganache.env --name concordia-ganache --net=concordia_concordia_network ecentrics/concordia-ganache:latest + @docker run -d -v ${GANACHE_VOLUMES}/ganache_keys:/mnt/concordia/ganache_keys -p 8545:8545 --env-file=./env/ganache.env --name concordia-ganache --net=concordia_concordia_network ecentrics/concordia-ganache:latest run-ganache-test: @docker run --rm -d -p 8546:8546 --env-file=./env/ganache.test.env --name concordia-ganache-test --net=concordia_concordia_network ecentrics/concordia-ganache:latest @@ -42,15 +42,15 @@ build-contracts-migrate: build-contracts-tests: @docker build ../ -f ./concordia-contracts/Dockerfile --target test -t ecentrics/concordia-contracts-tests --build-arg TZ=Europe/Athens run-contracts-tests: - @docker run --rm -v ${REPORTS}/contracts/:/usr/test-reports/ --env-file=./env/contracts.env --net=concordia_concordia_network ecentrics/concordia-contracts-tests:latest + @docker run --rm -v ${REPORTS}/contracts/:/mnt/concordia/test-reports/ --env-file=./env/contracts.env --net=concordia_concordia_network ecentrics/concordia-contracts-tests:latest run-contracts-tests-host-chain: - @docker run --rm -v ${REPORTS}/contracts/:/usr/test-reports/ --env-file=./env/contracts.env --net=host ecentrics/concordia-contracts-tests:latest + @docker run --rm -v ${REPORTS}/contracts/:/mnt/concordia/test-reports/ --env-file=./env/contracts.env --net=host ecentrics/concordia-contracts-tests:latest run-contracts-migrate: @docker run --rm -v ${PACKAGES}/concordia-contracts/build/:/usr/src/concordia/packages/concordia-contracts/build/ --env-file=./env/contracts.env --net=concordia_concordia_network ecentrics/concordia-contracts-migrate:latest run-contracts-migrate-host-chain: @docker run --rm -v ${PACKAGES}/concordia-contracts/build/:/usr/src/concordia/packages/concordia-contracts/build/ --env-file=./env/contracts.env --net=host ecentrics/concordia-contracts-migrate:latest get-contracts: - @docker run --rm -v ${PACKAGES}/concordia-contracts/build/:/build --entrypoint=sh ecentrics/concordia-contracts:latest -c 'cp /usr/src/concordia/packages/concordia-contracts/build/* /build' + @docker run --rm -v ${PACKAGES}/concordia-contracts/build/:/mnt/concordia/build --entrypoint=sh ecentrics/concordia-contracts:latest -c 'cp /usr/src/concordia/packages/concordia-contracts/build/* /mnt/concordia/build' # App targets build-app: @@ -60,7 +60,7 @@ build-app-staging: build-app-tests: @docker build ../ -f ./concordia-app/Dockerfile --target test -t ecentrics/concordia-app-tests --build-arg TZ=Europe/Athens run-app-tests: - @docker run --rm -v ${REPORTS}/app/:/usr/test-reports/ --env-file=./env/concordia.env ecentrics/concordia-app-tests:latest + @docker run --rm -v ${REPORTS}/app/:/mnt/concordia/test-reports/ --env-file=./env/concordia.env ecentrics/concordia-app-tests:latest run-app: @docker run -d --env-file=./env/concordia.env -p 7777:80 --name concordia-app ecentrics/concordia-app:latest run-app-staging: @@ -72,21 +72,21 @@ run-app-host-chain: build-contracts-provider: @docker build ../ -f ./concordia-contracts-provider/Dockerfile -t ecentrics/concordia-contracts-provider --build-arg TZ=Europe/Athens run-contracts-provider-staging: - @docker run -d -v ${CONTRACTS_PROVIDER_VOLUMES}/contracts:/data/contracts --env-file=./env/contracts-provider.env -e NODE_ENV=production -p 8400:8400 --name concordia-contracts-provider --net=concordia_concordia_network ecentrics/concordia-contracts-provider:latest + @docker run -d -v ${CONTRACTS_PROVIDER_VOLUMES}/contracts:/mnt/concordia/contracts --env-file=./env/contracts-provider.env -e NODE_ENV=production -p 8400:8400 --name concordia-contracts-provider --net=concordia_concordia_network ecentrics/concordia-contracts-provider:latest run-contracts-provider: - @docker run -d -v ${CONTRACTS_PROVIDER_VOLUMES}/contracts:/data/contracts --env-file=./env/contracts-provider.env -p 8400:8400 --name concordia-contracts-provider --net=concordia_concordia_network ecentrics/concordia-contracts-provider:latest + @docker run -d -v ${CONTRACTS_PROVIDER_VOLUMES}/contracts:/mnt/concordia/contracts --env-file=./env/contracts-provider.env -p 8400:8400 --name concordia-contracts-provider --net=concordia_concordia_network ecentrics/concordia-contracts-provider:latest # Pinner targets build-pinner: @docker build ../ -f ./concordia-pinner/Dockerfile -t ecentrics/concordia-pinner --build-arg TZ=Europe/Athens run-pinner-staging: - @docker run -d -v ${PINNER_VOLUMES}/orbitdb:/data/orbitdb --env-file=./env/pinner.env -e NODE_ENV=production -p 4444:4444 --name concordia-pinner --net=concordia_concordia_network ecentrics/concordia-pinner:latest + @docker run -d -v ${PINNER_VOLUMES}/orbitdb:/mnt/concordia/orbitdb --env-file=./env/pinner.env -e NODE_ENV=production -p 4444:4444 --name concordia-pinner --net=concordia_concordia_network ecentrics/concordia-pinner:latest run-pinner: - @docker run -d -v ${PINNER_VOLUMES}/orbitdb:/data/orbitdb --env-file=./env/pinner.env -p 4444:4444 --name concordia-pinner --net=concordia_concordia_network ecentrics/concordia-pinner:latest + @docker run -d -v ${PINNER_VOLUMES}/orbitdb:/mnt/concordia/orbitdb --env-file=./env/pinner.env -p 4444:4444 --name concordia-pinner --net=concordia_concordia_network ecentrics/concordia-pinner:latest run-pinner-staging-host: - @docker run -d -v ${PINNER_VOLUMES}/orbitdb:/data/orbitdb --env-file=./env/pinner.env -e NODE_ENV=production --net=host --name concordia-pinner ecentrics/concordia-pinner:latest + @docker run -d -v ${PINNER_VOLUMES}/orbitdb:/mnt/concordia/orbitdb --env-file=./env/pinner.env -e NODE_ENV=production --net=host --name concordia-pinner ecentrics/concordia-pinner:latest run-pinner-host: - @docker run -d -v ${PINNER_VOLUMES}/orbitdb:/data/orbitdb --env-file=./env/pinner.env --net=host --name concordia-pinner ecentrics/concordia-pinner:latest + @docker run -d -v ${PINNER_VOLUMES}/orbitdb:/mnt/concordia/orbitdb --env-file=./env/pinner.env --net=host --name concordia-pinner ecentrics/concordia-pinner:latest # Other clean-images: diff --git a/docker/concordia-app/Dockerfile b/docker/concordia-app/Dockerfile index 132ec8b..6673dbb 100755 --- a/docker/concordia-app/Dockerfile +++ b/docker/concordia-app/Dockerfile @@ -1,7 +1,7 @@ # -------------------------------------------------- # Stage 1 (Init application build base) # -------------------------------------------------- -FROM node:14-buster as base +FROM node:14-alpine@sha256:51e341881c2b77e52778921c685e711a186a71b8c6f62ff2edfc6b6950225a2f as base LABEL maintainers.1="Apostolos Fanakis " LABEL maintainers.2="Panagiotis Nikolaidis " LABEL gr.thmmy.ecentrics.concordia-image.name="app" @@ -17,6 +17,9 @@ COPY ./packages/concordia-contracts/package.json ./packages/concordia-contracts/ COPY ./packages/concordia-shared/package.json ./packages/concordia-shared/package.json COPY ./packages/concordia-app/package.json ./packages/concordia-app/ +# Install required packages +RUN apk update && apk --no-cache add g++ make python + RUN yarn install --frozen-lockfile # Gets the rest of the source code @@ -65,7 +68,7 @@ ENTRYPOINT ["yarn", "start"] # -------------------------------------------------- # Stage 4 (Production runtime) # -------------------------------------------------- -FROM nginx:1.17-alpine as production +FROM nginx:1.17-alpine@sha256:01747306a7247dbe928db991eab42e4002118bf636dd85b4ffea05dd907e5b66 as production LABEL maintainers.1="Apostolos Fanakis " LABEL maintainers.2="Panagiotis Nikolaidis /usr/test-reports/concordia-contracts-solhint.report) && - (yarn test --network env >/usr/test-reports/concordia-contracts-truffle-tests.report) +yarn _eslint -f html -o /mnt/concordia/test-reports/concordia-contracts-eslint.html --no-color && + (yarn _solhint >/mnt/concordia/test-reports/concordia-contracts-solhint.report) && + (yarn test --network env >/mnt/concordia/test-reports/concordia-contracts-truffle-tests.report) if [ $? -eq 0 ]; then echo "TESTS RAN SUCCESSFULLY!" diff --git a/docker/concordia-pinner/Dockerfile b/docker/concordia-pinner/Dockerfile index 3a2bf8b..3c9389c 100644 --- a/docker/concordia-pinner/Dockerfile +++ b/docker/concordia-pinner/Dockerfile @@ -1,7 +1,7 @@ # -------------------------------------------------- # Stage 1 (Runtime) # -------------------------------------------------- -FROM node:14-buster as runtime +FROM node:14-buster@sha256:32362e2ea89c62d77c86c8f26ad936dbbdc170cd6c06c4d7ff7a809012bb0c32 as runtime LABEL maintainers.1="Apostolos Fanakis " LABEL maintainers.2="Panagiotis Nikolaidis " LABEL gr.thmmy.ecentrics.concordia-image.name="pinner" diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 9ba19a1..db5f1ea 100755 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -15,7 +15,7 @@ services: - 8545:8545 user: root volumes: - - ./ganache/volumes/ganache_keys:/home/ganache_keys + - ./ganache/volumes/ganache_keys:/mnt/concordia/ganache_keys networks: concordia_network: restart: always diff --git a/docker/env/contracts-provider.env b/docker/env/contracts-provider.env index 6903a65..17f9e0d 100644 --- a/docker/env/contracts-provider.env +++ b/docker/env/contracts-provider.env @@ -1 +1 @@ -UPLOAD_CONTRACTS_DIRECTORY=/data/contracts/ +UPLOAD_CONTRACTS_DIRECTORY=/mnt/concordia/contracts/ diff --git a/docker/env/pinner.env b/docker/env/pinner.env index 84fb01e..e3ebe65 100644 --- a/docker/env/pinner.env +++ b/docker/env/pinner.env @@ -1,5 +1,5 @@ USE_EXTERNAL_CONTRACTS_PROVIDER=true -ORBIT_DIRECTORY=/data/orbitdb +ORBIT_DIRECTORY=/mnt/concordia/orbitdb CONTRACTS_PROVIDER_HOST=concordia-contracts-provider CONTRACTS_PROVIDER_PORT=8400 diff --git a/docker/ganache/Dockerfile b/docker/ganache/Dockerfile index 80eba88..11e7f39 100644 --- a/docker/ganache/Dockerfile +++ b/docker/ganache/Dockerfile @@ -1,4 +1,4 @@ -FROM trufflesuite/ganache-cli:latest +FROM trufflesuite/ganache-cli:v6.12.2@sha256:c062707f17f355872d703cde3de6a12fc45a027ed42857c72514171a5f466ab7 RUN mkdir /home/ganache_db /home/ganache_keys diff --git a/docker/ganache/start-blockchain.sh b/docker/ganache/start-blockchain.sh index 36aceea..0a373b8 100644 --- a/docker/ganache/start-blockchain.sh +++ b/docker/ganache/start-blockchain.sh @@ -14,8 +14,8 @@ if [ -z "${MNEMONIC}" ]; then --host "$HOST" \ --port "$PORT" \ --networkId "$ID" \ - --account_keys_path "/home/ganache_keys/keys.json" \ - --db "/home/ganache_db/" \ + --account_keys_path "/mnt/concordia/ganache_keys/keys.json" \ + --db "/mnt/concordia/ganache_db/" \ --allowUnlimitedContractSize \ --noVMErrorsOnRPCResponse \ --verbose @@ -28,8 +28,8 @@ else --host "$HOST" \ --port "$PORT" \ --networkId "$ID" \ - --account_keys_path "/home/ganache_keys/keys.json" \ - --db "/home/ganache_db/" \ + --account_keys_path "/mnt/concordia/ganache_keys/keys.json" \ + --db "/mnt/concordia/ganache_db/" \ --allowUnlimitedContractSize \ --noVMErrorsOnRPCResponse \ --deterministic \ From 9a26e6248adfcfb974a6e4bd0c0052d78ef6fb7a Mon Sep 17 00:00:00 2001 From: apostolof Date: Sat, 13 Feb 2021 16:25:36 +0200 Subject: [PATCH 13/18] refactor: add support for runtime environment variable in built app --- .dockerignore | 2 ++ docker/concordia-app/Dockerfile | 11 ++++++++++- docker/concordia-app/create-environment.sh | 14 ++++++++++++++ docker/concordia-app/run.sh | 3 +++ packages/concordia-app/public/index.html | 4 ++++ .../concordia-app/src/options/drizzleOptions.js | 4 +++- packages/concordia-app/src/utils/drizzleUtils.js | 8 ++++++-- .../src/environment/contractsProviderEnv.js | 6 ++++++ .../concordia-shared/src/environment/pinnerEnv.js | 11 +++++++++-- .../src/environment/rendezvousEnv.js | 11 +++++++++-- .../concordia-shared/src/environment/web3Env.js | 7 +++++++ 11 files changed, 73 insertions(+), 8 deletions(-) mode change 100644 => 100755 .dockerignore create mode 100755 docker/concordia-app/create-environment.sh create mode 100755 docker/concordia-app/run.sh diff --git a/.dockerignore b/.dockerignore old mode 100644 new mode 100755 index 46f9cac..c51f4a8 --- a/.dockerignore +++ b/.dockerignore @@ -8,6 +8,8 @@ docker/ !docker/concordia-contracts/test-contracts.sh !docker/concordia-app/test-app.sh !docker/concordia-app/nginx.conf +!docker/concordia-app/create-environment.sh +!docker/concordia-app/run.sh !docker/ganache/start-blockchain.sh packages/*/node_modules diff --git a/docker/concordia-app/Dockerfile b/docker/concordia-app/Dockerfile index 6673dbb..4ba629c 100755 --- a/docker/concordia-app/Dockerfile +++ b/docker/concordia-app/Dockerfile @@ -82,7 +82,16 @@ RUN apk add -U tzdata \ && apk del tzdata \ && rm -rf /var/cache/apk/* -WORKDIR "/var/www/concordia-app" +COPY ./docker/concordia-app/create-environment.sh /opt/concordia/create-environment.sh +COPY ./docker/concordia-app/run.sh /opt/concordia/run.sh + +RUN ["chmod", "+x", "/opt/concordia/create-environment.sh"] +RUN ["chmod", "+x", "/opt/concordia/run.sh"] COPY ./docker/concordia-app/nginx.conf /etc/nginx/conf.d/default.conf + +WORKDIR "/var/www/concordia-app" + COPY --chown=nginx:nginx --from=build /usr/src/concordia/packages/concordia-app/build . + +CMD ["/opt/concordia/run.sh"] diff --git a/docker/concordia-app/create-environment.sh b/docker/concordia-app/create-environment.sh new file mode 100755 index 0000000..c089455 --- /dev/null +++ b/docker/concordia-app/create-environment.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +echo "window.runtimeEnv = { \ +REACT_APP_CONCORDIA_HOST: \"${REACT_APP_CONCORDIA_HOST}\", \ +REACT_APP_CONCORDIA_PORT: \"${REACT_APP_CONCORDIA_PORT}\", \ +REACT_APP_WEB3_HOST: \"${REACT_APP_WEB3_HOST}\", \ +REACT_APP_WEB3_PORT: \"${REACT_APP_WEB3_PORT}\", \ +REACT_APP_RENDEZVOUS_HOST: \"${REACT_APP_RENDEZVOUS_HOST}\", \ +REACT_APP_RENDEZVOUS_PORT: \"${REACT_APP_RENDEZVOUS_PORT}\", \ +REACT_APP_USE_EXTERNAL_CONTRACTS_PROVIDER: \"${REACT_APP_USE_EXTERNAL_CONTRACTS_PROVIDER}\", \ +REACT_APP_CONTRACTS_PROVIDER_HOST: \"${REACT_APP_CONTRACTS_PROVIDER_HOST}\", \ +REACT_APP_CONTRACTS_PROVIDER_PORT: \"${REACT_APP_CONTRACTS_PROVIDER_PORT}\", \ +REACT_APP_CONTRACTS_VERSION_HASH: \"${REACT_APP_CONTRACTS_VERSION_HASH}\", \ +}" >/var/www/concordia-app/environment.js diff --git a/docker/concordia-app/run.sh b/docker/concordia-app/run.sh new file mode 100755 index 0000000..e0c073e --- /dev/null +++ b/docker/concordia-app/run.sh @@ -0,0 +1,3 @@ +sh /opt/concordia/create-environment.sh + +exec "$(which nginx)" -g "daemon off;" diff --git a/packages/concordia-app/public/index.html b/packages/concordia-app/public/index.html index 99d6395..cfd9385 100644 --- a/packages/concordia-app/public/index.html +++ b/packages/concordia-app/public/index.html @@ -20,6 +20,10 @@ Learn how to configure a non-root public URL by running `npm run build`. --> Concordia + + diff --git a/packages/concordia-app/src/options/drizzleOptions.js b/packages/concordia-app/src/options/drizzleOptions.js index ba6b211..81536fc 100644 --- a/packages/concordia-app/src/options/drizzleOptions.js +++ b/packages/concordia-app/src/options/drizzleOptions.js @@ -12,7 +12,9 @@ const drizzleOptions = { reloadWindowOnAccountChange: true, // We need it to reinitialize breeze and create new Orbit databases }; -if (process.env.REACT_APP_USE_EXTERNAL_CONTRACTS_PROVIDER) { +if (process.env.REACT_APP_USE_EXTERNAL_CONTRACTS_PROVIDER + || (window.runtimeEnv && window.runtimeEnv.REACT_APP_USE_EXTERNAL_CONTRACTS_PROVIDER)) { + console.log('Downloading contracts from external provider'); drizzleOptions.contracts = downloadContractArtifactsSync(); } else { drizzleOptions.contracts = contracts; diff --git a/packages/concordia-app/src/utils/drizzleUtils.js b/packages/concordia-app/src/utils/drizzleUtils.js index 59208f2..269d89f 100755 --- a/packages/concordia-app/src/utils/drizzleUtils.js +++ b/packages/concordia-app/src/utils/drizzleUtils.js @@ -10,8 +10,12 @@ import { } from '../constants/configuration/defaults'; function getContractsDownloadRequest() { - const HOST = process.env.REACT_APP_CONCORDIA_HOST || CONCORDIA_HOST_DEFAULT; - const PORT = process.env.REACT_APP_CONCORDIA_PORT || CONCORDIA_PORT_DEFAULT; + const HOST = process.env.REACT_APP_CONCORDIA_HOST + || (window.runtimeEnv && window.runtimeEnv.REACT_APP_CONCORDIA_HOST) + || CONCORDIA_HOST_DEFAULT; + const PORT = process.env.REACT_APP_CONCORDIA_PORT + || (window.runtimeEnv && window.runtimeEnv.REACT_APP_CONCORDIA_PORT) + || CONCORDIA_PORT_DEFAULT; const xhrRequest = new XMLHttpRequest(); diff --git a/packages/concordia-shared/src/environment/contractsProviderEnv.js b/packages/concordia-shared/src/environment/contractsProviderEnv.js index 3a545ca..c8e8df4 100644 --- a/packages/concordia-shared/src/environment/contractsProviderEnv.js +++ b/packages/concordia-shared/src/environment/contractsProviderEnv.js @@ -1,10 +1,16 @@ // Depending on the package user (app in contrast to any of the other packages) the env var names should either include // the REACT_APP_ prefix or not + +const runtimeEnv = typeof window !== 'undefined' && window.runtimeEnv; + const contractsProviderHostEnv = process.env.REACT_APP_CONTRACTS_PROVIDER_HOST + || (runtimeEnv && runtimeEnv.REACT_APP_CONTRACTS_PROVIDER_HOST) || process.env.CONTRACTS_PROVIDER_HOST; const contractsProviderPortEnv = process.env.REACT_APP_CONTRACTS_PROVIDER_PORT + || (runtimeEnv && runtimeEnv.REACT_APP_CONTRACTS_PROVIDER_PORT) || process.env.CONTRACTS_PROVIDER_PORT; const contractsVersionHashEnv = process.env.REACT_APP_CONTRACTS_VERSION_HASH + || (runtimeEnv && runtimeEnv.REACT_APP_CONTRACTS_VERSION_HASH) || process.env.CONTRACTS_VERSION_HASH; module.exports = { diff --git a/packages/concordia-shared/src/environment/pinnerEnv.js b/packages/concordia-shared/src/environment/pinnerEnv.js index d1df0bf..b5faf8d 100644 --- a/packages/concordia-shared/src/environment/pinnerEnv.js +++ b/packages/concordia-shared/src/environment/pinnerEnv.js @@ -1,7 +1,14 @@ // Depending on the package user (app in contrast to any of the other packages) the env var names should either include // the REACT_APP_ prefix or not -const pinnerApiHostEnv = process.env.REACT_APP_PINNER_API_HOST || process.env.PINNER_API_HOST; -const pinnerApiPortEnv = process.env.REACT_APP_PINNER_API_PORT || process.env.PINNER_API_PORT; + +const runtimeEnv = typeof window !== 'undefined' && window.runtimeEnv; + +const pinnerApiHostEnv = process.env.REACT_APP_PINNER_API_HOST + || (runtimeEnv && runtimeEnv.REACT_APP_PINNER_API_HOST) + || process.env.PINNER_API_HOST; +const pinnerApiPortEnv = process.env.REACT_APP_PINNER_API_PORT + || (runtimeEnv && runtimeEnv.REACT_APP_PINNER_API_PORT) + || process.env.PINNER_API_PORT; module.exports = { pinnerApiHostEnv, diff --git a/packages/concordia-shared/src/environment/rendezvousEnv.js b/packages/concordia-shared/src/environment/rendezvousEnv.js index 44f8965..76071b2 100644 --- a/packages/concordia-shared/src/environment/rendezvousEnv.js +++ b/packages/concordia-shared/src/environment/rendezvousEnv.js @@ -1,7 +1,14 @@ // Depending on the package user (app in contrast to any of the other packages) the env var names should either include // the REACT_APP_ prefix or not -const rendezvousHostEnv = process.env.REACT_APP_RENDEZVOUS_HOST || process.env.RENDEZVOUS_HOST; -const rendezvousPortEnv = process.env.REACT_APP_RENDEZVOUS_PORT || process.env.RENDEZVOUS_PORT; + +const runtimeEnv = typeof window !== 'undefined' && window.runtimeEnv; + +const rendezvousHostEnv = process.env.REACT_APP_RENDEZVOUS_HOST + || (runtimeEnv && runtimeEnv.REACT_APP_RENDEZVOUS_HOST) + || process.env.RENDEZVOUS_HOST; +const rendezvousPortEnv = process.env.REACT_APP_RENDEZVOUS_PORT + || (runtimeEnv && runtimeEnv.REACT_APP_RENDEZVOUS_PORT) + || process.env.RENDEZVOUS_PORT; module.exports = { rendezvousHostEnv, diff --git a/packages/concordia-shared/src/environment/web3Env.js b/packages/concordia-shared/src/environment/web3Env.js index 7bc2041..6996197 100644 --- a/packages/concordia-shared/src/environment/web3Env.js +++ b/packages/concordia-shared/src/environment/web3Env.js @@ -1,8 +1,13 @@ // Depending on the package user (app in contrast to any of the other packages) the env var names should either include // the REACT_APP_ prefix or not + +const runtimeEnv = typeof window !== 'undefined' && window.runtimeEnv; + const web3HostEnv = process.env.REACT_APP_WEB3_HOST + || (runtimeEnv && runtimeEnv.REACT_APP_WEB3_HOST) || process.env.WEB3_HOST; const web3PortEnv = process.env.REACT_APP_WEB3_PORT + || (runtimeEnv && runtimeEnv.REACT_APP_WEB3_PORT) || process.env.WEB3_PORT; // Web3 test environment shouldn't be available to the react app @@ -10,8 +15,10 @@ const web3HostTestEnv = process.env.WEB3_HOST_TEST; const web3PortTestEnv = process.env.WEB3_PORT_TEST; const web3PortSocketTimeoutEnv = process.env.REACT_APP_WEB3_PORT_SOCKET_TIMEOUT + || (runtimeEnv && runtimeEnv.REACT_APP_WEB3_PORT_SOCKET_TIMEOUT) || process.env.WEB3_PORT_SOCKET_TIMEOUT; const web3PortSocketConnectMaxAttemptsEnv = process.env.REACT_APP_WEB3_PORT_SOCKET_CONNECT_MAX_ATTEMPTS + || (runtimeEnv && runtimeEnv.REACT_APP_WEB3_PORT_SOCKET_CONNECT_MAX_ATTEMPTS) || process.env.WEB3_PORT_SOCKET_CONNECT_MAX_ATTEMPTS; module.exports = { From 99d4db9753dfe054d9a9a760eca0b44bb3d6a378 Mon Sep 17 00:00:00 2001 From: apostolof Date: Sat, 13 Feb 2021 16:25:53 +0200 Subject: [PATCH 14/18] refactor: use env vars during migration in docker image --- docker/concordia-contracts/migrate.sh | 4 +++- packages/concordia-contracts/package.json | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) mode change 100644 => 100755 docker/concordia-contracts/migrate.sh diff --git a/docker/concordia-contracts/migrate.sh b/docker/concordia-contracts/migrate.sh old mode 100644 new mode 100755 index 20a813f..f44afa0 --- a/docker/concordia-contracts/migrate.sh +++ b/docker/concordia-contracts/migrate.sh @@ -3,4 +3,6 @@ export CHAIN_HOST="$DEPLOY_CHAIN_HOST" export CHAIN_PORT="$DEPLOY_CHAIN_PORT" -cd /usr/src/concordia/packages/concordia-contracts && yarn _migrate --network "${MIGRATE_NETWORK}" --reset && yarn upload +cd /usr/src/concordia/packages/concordia-contracts && + yarn _migrate --network "${MIGRATE_NETWORK}" --reset && + yarn _upload ${CONTRACTS_VERSION_HASH} ${CONTRACTS_VERSION_TAG} diff --git a/packages/concordia-contracts/package.json b/packages/concordia-contracts/package.json index 41cbd3a..e348d9c 100644 --- a/packages/concordia-contracts/package.json +++ b/packages/concordia-contracts/package.json @@ -15,7 +15,8 @@ "migrate-upload": "yarn _migrate --network develop && yarn upload", "migrate-reset-upload": "yarn _migrate --network develop --reset && yarn upload", "_migrate": "yarn truffle migrate", - "upload": "node ./utils/contractsProviderUtils.js ${npm_package_version}-dev latest" + "upload": "yarn _upload ${npm_package_version}-dev latest", + "_upload": "node ./utils/contractsProviderUtils.js" }, "dependencies": { "@openzeppelin/contracts": "~3.3.0", From a643b697f868abe49a7aba1128549483abff3562 Mon Sep 17 00:00:00 2001 From: apostolof Date: Sat, 13 Feb 2021 16:26:10 +0200 Subject: [PATCH 15/18] refactor: remove unused env file, fix image run Makefile targets --- docker/Makefile | 12 ++++++------ docker/env/concordia.docker.env | 7 ------- 2 files changed, 6 insertions(+), 13 deletions(-) delete mode 100644 docker/env/concordia.docker.env diff --git a/docker/Makefile b/docker/Makefile index 3ddded8..e707706 100755 --- a/docker/Makefile +++ b/docker/Makefile @@ -72,21 +72,21 @@ run-app-host-chain: build-contracts-provider: @docker build ../ -f ./concordia-contracts-provider/Dockerfile -t ecentrics/concordia-contracts-provider --build-arg TZ=Europe/Athens run-contracts-provider-staging: - @docker run -d -v ${CONTRACTS_PROVIDER_VOLUMES}/contracts:/mnt/concordia/contracts --env-file=./env/contracts-provider.env -e NODE_ENV=production -p 8400:8400 --name concordia-contracts-provider --net=concordia_concordia_network ecentrics/concordia-contracts-provider:latest -run-contracts-provider: @docker run -d -v ${CONTRACTS_PROVIDER_VOLUMES}/contracts:/mnt/concordia/contracts --env-file=./env/contracts-provider.env -p 8400:8400 --name concordia-contracts-provider --net=concordia_concordia_network ecentrics/concordia-contracts-provider:latest +run-contracts-provider: + @docker run -d -v ${CONTRACTS_PROVIDER_VOLUMES}/contracts:/mnt/concordia/contracts --env-file=./env/contracts-provider.env -e NODE_ENV=production -p 8400:8400 --name concordia-contracts-provider --net=concordia_concordia_network ecentrics/concordia-contracts-provider:latest # Pinner targets build-pinner: @docker build ../ -f ./concordia-pinner/Dockerfile -t ecentrics/concordia-pinner --build-arg TZ=Europe/Athens run-pinner-staging: - @docker run -d -v ${PINNER_VOLUMES}/orbitdb:/mnt/concordia/orbitdb --env-file=./env/pinner.env -e NODE_ENV=production -p 4444:4444 --name concordia-pinner --net=concordia_concordia_network ecentrics/concordia-pinner:latest -run-pinner: @docker run -d -v ${PINNER_VOLUMES}/orbitdb:/mnt/concordia/orbitdb --env-file=./env/pinner.env -p 4444:4444 --name concordia-pinner --net=concordia_concordia_network ecentrics/concordia-pinner:latest +run-pinner: + @docker run -d -v ${PINNER_VOLUMES}/orbitdb:/mnt/concordia/orbitdb --env-file=./env/pinner.env -e NODE_ENV=production -p 4444:4444 --name concordia-pinner --net=concordia_concordia_network ecentrics/concordia-pinner:latest run-pinner-staging-host: - @docker run -d -v ${PINNER_VOLUMES}/orbitdb:/mnt/concordia/orbitdb --env-file=./env/pinner.env -e NODE_ENV=production --net=host --name concordia-pinner ecentrics/concordia-pinner:latest -run-pinner-host: @docker run -d -v ${PINNER_VOLUMES}/orbitdb:/mnt/concordia/orbitdb --env-file=./env/pinner.env --net=host --name concordia-pinner ecentrics/concordia-pinner:latest +run-pinner-host: + @docker run -d -v ${PINNER_VOLUMES}/orbitdb:/mnt/concordia/orbitdb --env-file=./env/pinner.env -e NODE_ENV=production --net=host --name concordia-pinner ecentrics/concordia-pinner:latest # Other clean-images: diff --git a/docker/env/concordia.docker.env b/docker/env/concordia.docker.env deleted file mode 100644 index 1810795..0000000 --- a/docker/env/concordia.docker.env +++ /dev/null @@ -1,7 +0,0 @@ -# Variables needed in runtime (in browser) -REACT_APP_RENDEZVOUS_HOST=rendezvous -REACT_APP_RENDEZVOUS_PORT=9090 - -# If the rendezvous server is running on host use these instead -#REACT_APP_RENDEZVOUS_HOST=127.0.0.1 -#REACT_APP_RENDEZVOUS_PORT=9090 From a2fafd7dbb493ba60cbde0e10f35dbc6a0432c84 Mon Sep 17 00:00:00 2001 From: apostolof Date: Sat, 13 Feb 2021 16:45:33 +0200 Subject: [PATCH 16/18] refactor: add helper scripts --- jenkins/check_package_changed.sh | 24 ++++++++++++++++++++++++ jenkins/hash_build_properties.sh | 18 ++++++++++++++++++ jenkins/map_to_thousand.sh | 5 +++++ 3 files changed, 47 insertions(+) create mode 100755 jenkins/check_package_changed.sh create mode 100755 jenkins/hash_build_properties.sh create mode 100755 jenkins/map_to_thousand.sh diff --git a/jenkins/check_package_changed.sh b/jenkins/check_package_changed.sh new file mode 100755 index 0000000..b915e37 --- /dev/null +++ b/jenkins/check_package_changed.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# Based on this post: +# https://engineering.brigad.co/only-deploy-services-impacted-by-changes-in-a-mono-repository-18f54b8ac109 + +APP=$1 +# Jenkins should provide these in the environment by default +GIT_COMMIT=$2 +GIT_PREVIOUS_COMMIT=$3 +ROOT_FILES_AND_FOLDERS=${4:-"package.json" "yarn.lock" ".dockerignore" "docker" "jenkins"} + +function join_by { local IFS="$1"; shift; echo "$*"; } + +function package_changed { git diff --name-only "$COMMIT_RANGE" | grep -qE "^packages/$1/" && echo true || echo false; } + +COMMIT_RANGE="$GIT_PREVIOUS_COMMIT...$GIT_COMMIT" +ROOT_FILES_AND_FOLDERS_ARRAY=($ROOT_FILES_AND_FOLDERS) +ROOT_FILES_AND_FOLDERS_JOINED=$(join_by "|" ${ROOT_FILES_AND_FOLDERS_ARRAY[*]}) + +ROOT_FILES_CHANGED=$(git diff --name-only "$COMMIT_RANGE" | grep -qE "^($ROOT_FILES_AND_FOLDERS_JOINED)" && echo true || echo false) +IS_FORCE_BUILD=$(git log --oneline "$COMMIT_RANGE" | grep -qE "ci: force" && echo true || echo false) +APP_FILES_CHANGED=$(package_changed ${APP}) + +($IS_FORCE_BUILD || $ROOT_FILES_CHANGED || $APP_FILES_CHANGED) && echo 0 || echo 1 diff --git a/jenkins/hash_build_properties.sh b/jenkins/hash_build_properties.sh new file mode 100755 index 0000000..a30e81c --- /dev/null +++ b/jenkins/hash_build_properties.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# Outputs to the stdout a deterministically generated integer in the range 0-4095. The integer is generated using the +# input strings and SHA1. +# Usage: hash_build_properties.sh +# Inputs: +# - branch: the branch being build +# - build_number the incrementing number of the build + +BRANCH=$1 +BUILD_NUMBER=$2 + +STRING_TO_HASH="$BRANCH-$BUILD_NUMBER" +SHA1_SUM_HEX=$(sha1sum <<<"$STRING_TO_HASH") +SHA1_TRUNCATED_HEX=$(cut -c1-3 <<<"$SHA1_SUM_HEX") +HASHED_STRING=$((0x${SHA1_TRUNCATED_HEX})) + +echo "$HASHED_STRING" diff --git a/jenkins/map_to_thousand.sh b/jenkins/map_to_thousand.sh new file mode 100755 index 0000000..d94d29d --- /dev/null +++ b/jenkins/map_to_thousand.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +INTEGER_TO_MAP=$1 + +echo $(( INTEGER_TO_MAP * 999 / 4095 )) From 2388090add8c54be075968f8588e0865e347cd54 Mon Sep 17 00:00:00 2001 From: apostolof Date: Sat, 13 Feb 2021 17:02:35 +0200 Subject: [PATCH 17/18] feat: add ci-cd configuration --- jenkins/Jenkinsfile | 739 ++++++++++++++++++ jenkins/env/concordia.production.jenkins.env | 16 + jenkins/env/concordia.staging.jenkins.env | 16 + jenkins/env/contracts.production.jenkins.env | 8 + jenkins/env/contracts.provider.production.env | 8 + jenkins/env/contracts.provider.staging.env | 8 + jenkins/env/contracts.staging.jenkins.env | 8 + jenkins/env/contracts.test.jenkins.env | 4 + jenkins/env/ganache.production.jenkins.env | 10 + jenkins/env/ganache.staging.jenkins.env | 10 + jenkins/env/ganache.test.jenkins.env | 6 + jenkins/env/pinner.production.jenkins.env | 20 + jenkins/env/pinner.staging.jenkins.env | 20 + 13 files changed, 873 insertions(+) create mode 100755 jenkins/Jenkinsfile create mode 100755 jenkins/env/concordia.production.jenkins.env create mode 100755 jenkins/env/concordia.staging.jenkins.env create mode 100755 jenkins/env/contracts.production.jenkins.env create mode 100755 jenkins/env/contracts.provider.production.env create mode 100755 jenkins/env/contracts.provider.staging.env create mode 100755 jenkins/env/contracts.staging.jenkins.env create mode 100755 jenkins/env/contracts.test.jenkins.env create mode 100755 jenkins/env/ganache.production.jenkins.env create mode 100755 jenkins/env/ganache.staging.jenkins.env create mode 100755 jenkins/env/ganache.test.jenkins.env create mode 100755 jenkins/env/pinner.production.jenkins.env create mode 100755 jenkins/env/pinner.staging.jenkins.env diff --git a/jenkins/Jenkinsfile b/jenkins/Jenkinsfile new file mode 100755 index 0000000..d6aab72 --- /dev/null +++ b/jenkins/Jenkinsfile @@ -0,0 +1,739 @@ +#!groovy + +def cleanSlateEnabled + +// Package change state +def appPackageChanged +def contractsPackageChanged +def contractsProviderPackageChanged +def pinnerPackageChanged +def sharedPackageChanged + +// Package versions +def appPackageVersion +def contractsPackageVersion +def contractsProviderPackageVersion +def pinnerPackageVersion +def sharedPackageVersion + +// Docker images +def appImage +def appTestsImage +def contractsImage +def contractsTestsImage +def contractsProviderImage +def pinnerImage + +def freshGanacheStagingRunning +def freshGanacheProductionRunning + +pipeline { + agent any + + post { + failure { + updateGitlabCommitStatus name: 'build', state: 'failed' + } + success { + updateGitlabCommitStatus name: 'build', state: 'success' + } + always { + archiveArtifacts artifacts: "reports/${BUILD_NUMBER}/**/* , build/**/*, ganache/*", fingerprint: true + sleep 2 + sh 'docker images | grep -E "ecentrics/concordia.+tests" | tr -s \' \' | cut -d \' \' -f 3 | xargs --no-run-if-empty docker rmi -f || true' + sh 'docker images | grep -E "ecentrics/concordia.+staging" | tr -s \' \' | cut -d \' \' -f 3 | xargs --no-run-if-empty docker rmi -f || true' + sh 'docker system prune -f' + sh 'rm -rf reports' + sh 'rm -rf build' + } + } + options { + gitLabConnection('apella') + } + triggers { + gitlab(triggerOnPush: true, triggerOnMergeRequest: true, branchFilterType: 'All') + } + environment { + DOCKER_BUILDKIT='1' + } + + stages { + stage ('VERSION') { + steps { + script { + cleanSlateEnabled = sh (script: "git log -1 | grep -qE 'ci: force'", returnStatus: true) + + appPackageChanged = sh(script: 'bash ./jenkins/check_package_changed.sh app "$GIT_COMMIT" "$GIT_PREVIOUS_COMMIT"', returnStdout: true).trim() + contractsPackageChanged = sh(script: 'bash ./jenkins/check_package_changed.sh contracts "$GIT_COMMIT" "$GIT_PREVIOUS_COMMIT"', returnStdout: true).trim() + contractsProviderPackageChanged = sh(script: 'bash ./jenkins/check_package_changed.sh concordia-contracts-provider "$GIT_COMMIT" "$GIT_PREVIOUS_COMMIT"', returnStdout: true).trim() + pinnerPackageChanged = sh(script: 'bash ./jenkins/check_package_changed.sh pinner "$GIT_COMMIT" "$GIT_PREVIOUS_COMMIT"', returnStdout: true).trim() + sharedPackageChanged = sh(script: 'bash ./jenkins/check_package_changed.sh shared "$GIT_COMMIT" "$GIT_PREVIOUS_COMMIT"', returnStdout: true).trim() + + appPackageVersion = sh(script: 'grep "\\"version\\":" ./packages/concordia-app/package.json | head -1 | awk -F: \'{ print $2 }\' | sed \'s/[",]//g\' | tr -d \'[[:space:]]\'', returnStdout: true).trim() + contractsPackageVersion = sh(script: 'grep "\\"version\\":" ./packages/concordia-contracts/package.json | head -1 | awk -F: \'{ print $2 }\' | sed \'s/[",]//g\' | tr -d \'[[:space:]]\'', returnStdout: true).trim() + contractsProviderPackageVersion = sh(script: 'grep "\\"version\\":" ./packages/concordia-contracts-provider/package.json | head -1 | awk -F: \'{ print $2 }\' | sed \'s/[",]//g\' | tr -d \'[[:space:]]\'', returnStdout: true).trim() + pinnerPackageVersion = sh(script: 'grep "\\"version\\":" ./packages/concordia-pinner/package.json | head -1 | awk -F: \'{ print $2 }\' | sed \'s/[",]//g\' | tr -d \'[[:space:]]\'', returnStdout: true).trim() + sharedPackageVersion = sh(script: 'grep "\\"version\\":" ./packages/concordia-shared/package.json | head -1 | awk -F: \'{ print $2 }\' | sed \'s/[",]//g\' | tr -d \'[[:space:]]\'', returnStdout: true).trim() + + echo "Package: app, Version: ${appPackageVersion}, Changed: ${appPackageChanged}" + echo "Package: contracts, Version: ${contractsPackageVersion}, Changed: ${contractsPackageChanged}" + echo "Package: contracts-provider, Version: ${contractsProviderPackageVersion}, Changed: ${contractsProviderPackageChanged}" + echo "Package: pinner, Version: ${pinnerPackageVersion}, Changed: ${pinnerPackageChanged}" + echo "Package: shared, Version: ${sharedPackageVersion}, Changed: ${sharedPackageChanged}" + } + } + } + + stage('TEST') { + parallel { + stage('TEST CONTRACTS') { + steps { + script { + try { + echo "${BRANCH_NAME}-${BUILD_NUMBER}" + def ganacheTestPort = sh(script: "bash ./jenkins/hash_build_properties.sh ${BRANCH_NAME} ${BUILD_NUMBER} | xargs bash ./jenkins/map_to_thousand.sh", returnStdout: true).trim() + echo "$ganacheTestPort" + + def ganacheTestImage = docker.build( + "ecentrics/concordia-ganache", + "-f docker/ganache/Dockerfile \ + ./" + ) + + sh 'docker network create --driver bridge concordia_ganache_test_network || true' + + ganacheTestImage.withRun("""-d -p 6${ganacheTestPort}:8546 \ + --env-file=./jenkins/env/ganache.test.jenkins.env \ + --name concordia-ganache-test-6${ganacheTestPort} \ + --net=concordia_ganache_test_network""") { concordiaGanacheTest -> + + contractsTestsImage = docker.build( + "ecentrics/concordia-contracts-tests:v${contractsPackageVersion}-b${BUILD_NUMBER}-tests", + "-f docker/concordia-contracts/Dockerfile \ + ./ \ + --target test \ + --build-arg TZ=Europe/Athens" + ) + + contractsTestsImage.run("""--rm \ + -v ecentrics_janus_common:/mnt/concordia/test-reports/ \ + --env-file=./jenkins/env/contracts.test.jenkins.env \ + -e WEB3_HOST=concordia-ganache-test-6${ganacheTestPort} \ + -e WEB3_PORT=6${ganacheTestPort} \ + --net=concordia_ganache_test_network""") + + sh 'mkdir -p ./reports/${BUILD_NUMBER}/contracts' + sh 'find /mnt/janus/common/ -name "concordia-contracts-*" -exec cp \'{}\' ./reports/${BUILD_NUMBER}/contracts/ \\;' + } + } catch (e) { + error('Some tests failed!') + error('Aborting the build.') + throw e + } + } + } + } + + stage('TEST APP') { + steps { + script { + appTestsImage = docker.build( + "ecentrics/concordia-app:v${appPackageVersion}-b${BUILD_NUMBER}-tests", + "-f docker/concordia-app/Dockerfile \ + ./ \ + --target test \ + --build-arg TZ=Europe/Athens" + ) + + try { + appTestsImage.run('--rm \ + -v ecentrics_janus_common:/mnt/concordia/test-reports/') + + sh 'mkdir -p ./reports/${BUILD_NUMBER}/app' + sh 'find /mnt/janus/common/ -name "concordia-app-*" -exec cp \'{}\' ./reports/${BUILD_NUMBER}/app/ \\;' + } catch (e) { + error('Some tests failed!') + error('Aborting the build.') + throw e + } + } + } + } + } + } + + stage('BUILD FOR PRODUCTION') { + when { + branch 'master' + } + parallel { + stage('BUILD CONTRACTS') { + when { + expression { + return "${contractsPackageChanged}" == '0' || "${sharedPackageChanged}" == '0'; + } + } + steps { + script { + contractsImage = docker.build( + "ecentrics/concordia-contracts-migrate:v${contractsPackageVersion}", + "-f docker/concordia-contracts/Dockerfile \ + ./ \ + --build-arg TZ=Europe/Athens" + ) + + contractsImage.run('--rm \ + -v ecentrics_janus_common:/mnt/concordia/build \ + --entrypoint=sh', + "-c 'mkdir -p /mnt/concordia/build/contract-artifacts && cp /usr/src/concordia/packages/concordia-contracts/build/* /mnt/concordia/build/contract-artifacts'") + + sh 'mkdir -p ./build/${BUILD_NUMBER}/contracts' + sh 'cp /mnt/janus/common/contract-artifacts/* ./build/${BUILD_NUMBER}/contracts' + } + } + } + + stage('BUILD APP') { + when { + expression { + return "${appPackageChanged}" == '0' || "${sharedPackageChanged}" == '0'; + } + } + steps { + script { + appImage = docker.build( + "ecentrics/concordia-app:v${appPackageVersion}", + "-f docker/concordia-app/Dockerfile \ + ./ \ + --build-arg TZ=Europe/Athens" + ) + } + } + } + + stage('BUILD CONTRACTS PROVIDER') { + when { + expression { + return "${contractsProviderPackageChanged}" == '0' || "${sharedPackageChanged}" == '0'; + } + } + steps { + script { + contractsProviderImage = docker.build( + "ecentrics/concordia-contracts-provider:v${contractsProviderPackageVersion}", + "-f docker/concordia-contracts-provider/Dockerfile \ + ./ \ + --build-arg TZ=Europe/Athens" + ) + } + } + } + + stage('BUILD PINNER') { + when { + expression { + return "${pinnerPackageChanged}" == '0' || "${sharedPackageChanged}" == '0'; + } + } + steps { + script { + pinnerImage = docker.build( + "ecentrics/concordia-pinner:v${pinnerPackageVersion}", + "-f docker/concordia-pinner/Dockerfile \ + ./ \ + --build-arg TZ=Europe/Athens" + ) + } + } + } + } + } + + stage('BUILD FOR STAGING') { + when { + branch 'develop' + } + parallel { + stage('BUILD CONTRACTS') { + when { + expression { + return "${contractsPackageChanged}" == '0' || "${sharedPackageChanged}" == '0'; + } + } + steps { + script { + contractsImage = docker.build( + "ecentrics/concordia-contracts-migrate:v${contractsPackageVersion}-staging-b${BUILD_NUMBER}", + "-f docker/concordia-contracts/Dockerfile \ + ./ \ + --build-arg TZ=Europe/Athens" + ) + + // Get contract artifacts + contractsImage.run('--rm \ + -v ecentrics_janus_common:/mnt/concordia/build \ + --entrypoint=sh', + "-c 'mkdir -p /mnt/concordia/build/contract-artifacts && cp /usr/src/concordia/packages/concordia-contracts/build/* /mnt/concordia/build/contract-artifacts'") + + sh 'mkdir -p ./build/${BUILD_NUMBER}/contracts' + sh 'cp /mnt/janus/common/contract-artifacts/* ./build/${BUILD_NUMBER}/contracts' + } + } + } + + stage('BUILD APP') { + when { + expression { + return "${appPackageChanged}" == '0' || "${sharedPackageChanged}" == '0'; + } + } + steps { + script { + appImage = docker.build( + "ecentrics/concordia-app:v${appPackageVersion}-staging-b${BUILD_NUMBER}", + "-f docker/concordia-app/Dockerfile \ + ./ \ + --target staging \ + --build-arg TZ=Europe/Athens" + ) + } + } + } + + stage('BUILD CONTRACTS PROVIDER') { + when { + expression { + return "${contractsProviderPackageChanged}" == '0' || "${sharedPackageChanged}" == '0'; + } + } + steps { + script { + contractsProviderImage = docker.build( + "ecentrics/concordia-contracts-provider:v${contractsProviderPackageVersion}-staging-b${BUILD_NUMBER}", + "-f docker/concordia-contracts-provider/Dockerfile \ + ./ \ + --build-arg TZ=Europe/Athens" + ) + } + } + } + + stage('BUILD PINNER') { + when { + expression { + return "${pinnerPackageChanged}" == '0' || "${sharedPackageChanged}" == '0'; + } + } + steps { + script { + pinnerImage = docker.build( + "ecentrics/concordia-pinner:v${pinnerPackageVersion}-staging-b${BUILD_NUMBER}", + "-f docker/concordia-pinner/Dockerfile \ + ./ \ + --build-arg TZ=Europe/Athens" + ) + } + } + } + } + } + + stage('PUBLISH') { + when { + branch 'master' + } + parallel { + stage('PUBLISH CONTRACTS') { + when { + expression { + return "${contractsPackageChanged}" == '0' || "${sharedPackageChanged}" == '0'; + } + } + steps { + script { + docker.withRegistry('https://registry.hub.docker.com/', 'docker-hub-concordia') { + contractsImage.push() + contractsImage.push('latest') + } + } + } + } + + stage('PUBLISH APP') { + when { + expression { + return "${appPackageChanged}" == '0' || "${sharedPackageChanged}" == '0'; + } + } + steps { + script { + docker.withRegistry('https://registry.hub.docker.com/', 'docker-hub-concordia') { + appImage.push() + appImage.push('latest') + } + } + } + } + + stage('PUBLISH CONTRACTS PROVIDER') { + when { + expression { + return "${contractsProviderPackageChanged}" == '0' || "${sharedPackageChanged}" == '0'; + } + } + steps { + script { + docker.withRegistry('https://registry.hub.docker.com/', 'docker-hub-concordia') { + contractsProviderImage.push() + contractsProviderImage.push('latest') + } + } + } + } + + stage('PUBLISH PINNER') { + when { + expression { + return "${pinnerPackageChanged}" == '0' || "${sharedPackageChanged}" == '0'; + } + } + steps { + script { + docker.withRegistry('https://registry.hub.docker.com/', 'docker-hub-concordia') { + pinnerImage.push() + pinnerImage.push('latest') + } + } + } + } + } + } + + stage('DEPLOY STAGING') { + when { + branch 'develop' + } + stages { + stage('STAGING DEPLOY PREPARATION') { + steps { + script { + sh 'docker network create --driver bridge ecentrics_concordia_staging_network || true' + + def rendezvousServerRunning = sh (script: 'docker ps -f name=concordia-rendezvous | \ + grep -qE concordia-rendezvous', returnStatus: true) + + if ("$rendezvousServerRunning" == '1') { + sh 'docker run \ + -d \ + -p 9090:9090 \ + --name concordia-rendezvous \ + --net=ecentrics_concordia_staging_network \ + libp2p/js-libp2p-webrtc-star:version-0.20.5' + } else { + sh 'docker network connect ecentrics_concordia_staging_network concordia-rendezvous || true' + } + } + } + } + + stage('DEPLOY CONTRACTS PROVIDER') { + when { + expression { + def contractsProviderStagingRunning = sh (script: 'docker ps -f name=concordia-contracts-provider-staging | \ + grep -qE concordia-contracts-provider-staging', returnStatus: true) + return "${contractsProviderPackageChanged}" == '0' || "$cleanSlateEnabled" == '0' || "$contractsProviderStagingRunning" == '1' || "${sharedPackageChanged}" == '0'; + } + } + steps { + script { + sh 'docker stop concordia-contracts-provider-staging || true \ + && docker rm concordia-contracts-provider-staging || true' + + sh 'if [ "$cleanSlateEnabled" -eq "0" ]; then \ + docker volume rm concordia-contracts-provider-staging || true; \ + fi' + + sh (script: """docker run \ + -d \ + -v concordia-contracts-provider-staging:/mnt/concordia/contracts \ + --env-file=./jenkins/env/contracts.provider.staging.env \ + -p 8450:8450 \ + --name concordia-contracts-provider-staging \ + --net=ecentrics_concordia_staging_network \ + ecentrics/concordia-contracts-provider:v${contractsProviderPackageVersion}-staging-b${BUILD_NUMBER}""") + } + } + } + + stage('RECREATE GANACHE') { + when { + expression { + def ganacheStagingRunning = sh (script: 'docker ps -f name=concordia-ganache-staging | \ + grep -qE concordia-ganache-staging', returnStatus: true) + return "$cleanSlateEnabled" == '0' || "$ganacheStagingRunning" == '1'; + } + } + steps { + script { + sh 'docker stop concordia-ganache-staging || true \ + && docker rm concordia-ganache-staging || true' + + sh 'docker volume rm concordia-ganache-staging || true' + + sh (script: 'docker run \ + -d \ + -v concordia-ganache-staging:/mnt/concordia/ganache_keys \ + -p 8555:8555 \ + --env-file=./jenkins/env/ganache.staging.jenkins.env \ + --name concordia-ganache-staging \ + --net=ecentrics_concordia_staging_network \ + ecentrics/concordia-ganache:latest') + + // Ganache image might take a while to come alive + sleep 10 + + sh 'mkdir -p ./ganache/ && docker cp concordia-ganache-staging:/mnt/concordia/ganache_keys/keys.json ./ganache/' + freshGanacheStagingRunning = true + } + } + } + + stage('DEPLOY CONTRACTS') { + when { + expression { + return "${contractsPackageChanged}" == '0' || "$cleanSlateEnabled" == '0' || "$freshGanacheStagingRunning" || "${sharedPackageChanged}" == '0'; + } + } + steps { + script { + sh """docker run \ + --rm \ + --env-file=./jenkins/env/contracts.staging.jenkins.env \ + -e CONTRACTS_VERSION_HASH=${contractsPackageVersion}-dev \ + --net=ecentrics_concordia_staging_network \ + ecentrics/concordia-contracts-migrate:v${contractsPackageVersion}-staging-b${BUILD_NUMBER}""" + } + } + } + + stage('DEPLOY PINNER') { + when { + expression { + def pinnerStagingRunning = sh (script: 'docker ps -f name=concordia-pinner-staging | \ + grep -qE concordia-pinner-staging', returnStatus: true) + return "${pinnerPackageChanged}" == '0' || "$cleanSlateEnabled" == '0' || "${contractsPackageChanged}" == '0' || "${sharedPackageChanged}" == '0'; + } + } + steps { + script { + sh 'docker stop concordia-pinner-staging || true \ + && docker rm concordia-pinner-staging || true' + + sh 'if [ "$cleanSlateEnabled" -eq "0" ]; then \ + docker volume rm concordia-pinner-staging || true; \ + fi' + + sh """docker run \ + -d \ + -v concordia-pinner-staging:/mnt/concordia/orbitdb \ + -p 5555:5555 \ + --env-file=./jenkins/env/pinner.staging.jenkins.env \ + --name concordia-pinner-staging \ + --net=ecentrics_concordia_staging_network \ + ecentrics/concordia-pinner:v${pinnerPackageVersion}-staging-b${BUILD_NUMBER}""" + } + } + } + + stage('DEPLOY APP') { + when { + expression { + def pinnerStagingRunning = sh (script: 'docker ps -f name=concordia-app-staging | \ + grep -qE concordia-app-staging', returnStatus: true) + return "${appPackageChanged}" == '0' || "$cleanSlateEnabled" == '0' || "${sharedPackageChanged}" == '0'; + } + } + + steps { + script { + sh 'docker stop concordia-app-staging || true \ + && docker rm concordia-app-staging || true' + + sh """docker run \ + -itd \ + -p 7000:3000 \ + --env-file=./jenkins/env/concordia.staging.jenkins.env \ + --name concordia-app-staging \ + ecentrics/concordia-app:v${appPackageVersion}-staging-b${BUILD_NUMBER}""" + } + } + } + } + } + + stage('DEPLOY PRODUCTION') { + when { + branch 'master' + } + stages { + stage('PRODUCTION DEPLOY PREPARATION') { + steps { + script { + sh 'docker network create --driver bridge ecentrics_concordia_production_network || true' + + def rendezvousServerRunning = sh (script: 'docker ps -f name=concordia-rendezvous | \ + grep -qE concordia-rendezvous', returnStatus: true) + + if ("$rendezvousServerRunning" == '1') { + sh 'docker run \ + -d \ + -p 9090:9090 \ + --name concordia-rendezvous \ + --net=ecentrics_concordia_production_network \ + libp2p/js-libp2p-webrtc-star:version-0.20.5' + } else { + sh 'docker network connect ecentrics_concordia_production_network concordia-rendezvous || true' + } + } + } + } + + stage('DEPLOY CONTRACTS PROVIDER') { + when { + expression { + def contractsProviderProductionRunning = sh (script: 'docker ps -f name=concordia-contracts-provider-production | \ + grep -qE concordia-contracts-provider-production', returnStatus: true) + return "${contractsProviderPackageChanged}" == '0' || "$cleanSlateEnabled" == '0' || "$contractsProviderProductionRunning" == '1' || "${sharedPackageChanged}" == '0'; + } + } + steps { + script { + sh 'docker stop concordia-contracts-provider-production || true \ + && docker rm concordia-contracts-provider-production || true' + + sh 'if [ "$cleanSlateEnabled" -eq "0" ]; then \ + docker volume rm concordia-contracts-provider-production || true; \ + fi' + + sh (script: """docker run \ + -d \ + -v concordia-contracts-provider-production:/mnt/concordia/contracts \ + --env-file=./jenkins/env/contracts.provider.production.env \ + -e NODE_ENV=production \ + -p 8400:8400 \ + --name concordia-contracts-provider-production \ + --net=ecentrics_concordia_production_network \ + ecentrics/concordia-contracts-provider:v${contractsProviderPackageVersion}""") + } + } + } + + stage('RECREATE GANACHE') { + when { + expression { + def ganacheProductionRunning = sh (script: 'docker ps -f name=concordia-ganache-production | \ + grep -qE concordia-ganache-production', returnStatus: true) + return "$cleanSlateEnabled" == '0' || "$ganacheProductionRunning" == '1'; + } + } + steps { + script { + sh 'docker stop concordia-ganache-production || true \ + && docker rm concordia-ganache-production || true' + + sh 'docker volume rm concordia-ganache-production || true' + + sh (script: 'docker run \ + -d \ + -v concordia-ganache-production:/mnt/concordia/ganache_keys \ + -p 8545:8545 \ + --env-file=./jenkins/env/ganache.production.jenkins.env \ + --name concordia-ganache-production \ + --net=ecentrics_concordia_production_network \ + ecentrics/concordia-ganache:latest') + + // Ganache image might take a while to come alive + sleep 10 + + sh 'mkdir -p ./ganache/ && docker cp concordia-ganache-production:/mnt/concordia/ganache_keys/keys.json ./ganache/' + freshGanacheProductionRunning = true + } + } + } + + stage('DEPLOY CONTRACTS') { + when { + expression { + return "${contractsPackageChanged}" == '0' || "$cleanSlateEnabled" == '0' || "$freshGanacheProductionRunning" || "${sharedPackageChanged}" == '0'; + } + } + steps { + script { + sh """docker run \ + --rm \ + --env-file=./jenkins/env/contracts.production.jenkins.env \ + -e CONTRACTS_VERSION_HASH=${contractsPackageVersion} \ + --net=ecentrics_concordia_production_network \ + ecentrics/concordia-contracts-migrate:v${contractsPackageVersion}""" + } + } + } + + stage('DEPLOY PINNER') { + when { + expression { + def pinnerProductionRunning = sh (script: 'docker ps -f name=concordia-pinner-production | \ + grep -qE concordia-pinner-production', returnStatus: true) + return "${pinnerPackageChanged}" == '0' || "$cleanSlateEnabled" == '0' || "${pinnerProductionRunning}" == '1' || "${contractsPackageChanged}" == '0' || "${sharedPackageChanged}" == '0'; + } + } + steps { + script { + sh 'docker stop concordia-pinner-production || true \ + && docker rm concordia-pinner-production || true' + + sh 'if [ "$cleanSlateEnabled" -eq "0" ]; then \ + docker volume rm concordia-pinner-production || true; \ + fi' + + sh """docker run \ + -d \ + -v concordia-pinner-production:/mnt/concordia/orbitdb \ + -p 4444:4444 \ + -e NODE_ENV=production \ + --env-file=./jenkins/env/pinner.production.jenkins.env \ + --name concordia-pinner-production \ + --net=ecentrics_concordia_production_network \ + ecentrics/concordia-pinner:v${pinnerPackageVersion}""" + } + } + } + + stage('DEPLOY APP') { + when { + expression { + def appProductionRunning = sh (script: 'docker ps -f name=concordia-app-production | \ + grep -qE concordia-app-production', returnStatus: true) + return "${appPackageChanged}" == '0' || "$cleanSlateEnabled" == '0' ||"${appProductionRunning}" == '1' || "${sharedPackageChanged}" == '0'; + } + } + + steps { + script { + sh 'docker stop concordia-app-production || true \ + && docker rm concordia-app-production || true' + + sh """docker run \ + -d \ + -p 7777:80 \ + --env-file=./jenkins/env/concordia.production.jenkins.env \ + --name concordia-app-production \ + ecentrics/concordia-app:v${appPackageVersion}""" + } + } + } + } + } + } +} + diff --git a/jenkins/env/concordia.production.jenkins.env b/jenkins/env/concordia.production.jenkins.env new file mode 100755 index 0000000..9dc3550 --- /dev/null +++ b/jenkins/env/concordia.production.jenkins.env @@ -0,0 +1,16 @@ +VIRTUAL_HOST=concordia.ecentrics.net +VIRTUAL_PORT=7777 +LETSENCRYPT_HOST=concordia.ecentrics.net +LETSENCRYPT_EMAIL=ecentricsgr@gmail.com + +# Variables needed in runtime (in browser) +REACT_APP_CONCORDIA_HOST=concordia.ecentrics.net +REACT_APP_CONCORDIA_PORT=7777 + +REACT_APP_RENDEZVOUS_HOST=rendezvous.ecentrics.net +REACT_APP_RENDEZVOUS_PORT=9090 + +REACT_APP_USE_EXTERNAL_CONTRACTS_PROVIDER=true +REACT_APP_CONTRACTS_PROVIDER_HOST=contracts.concordia.ecentrics.net +REACT_APP_CONTRACTS_PROVIDER_PORT=8400 +REACT_APP_CONTRACTS_VERSION_HASH=stable diff --git a/jenkins/env/concordia.staging.jenkins.env b/jenkins/env/concordia.staging.jenkins.env new file mode 100755 index 0000000..49278b2 --- /dev/null +++ b/jenkins/env/concordia.staging.jenkins.env @@ -0,0 +1,16 @@ +VIRTUAL_HOST=staging.concordia.ecentrics.net +VIRTUAL_PORT=7000 +LETSENCRYPT_HOST=staging.concordia.ecentrics.net +LETSENCRYPT_EMAIL=ecentricsgr@gmail.com + +# Variables needed in runtime (in browser) +REACT_APP_CONCORDIA_HOST=staging.concordia.ecentrics.net +REACT_APP_CONCORDIA_PORT=7000 + +REACT_APP_RENDEZVOUS_HOST=rendezvous.ecentrics.net +REACT_APP_RENDEZVOUS_PORT=9090 + +REACT_APP_USE_EXTERNAL_CONTRACTS_PROVIDER=true +REACT_APP_CONTRACTS_PROVIDER_HOST=staging.contracts.concordia.ecentrics.net +REACT_APP_CONTRACTS_PROVIDER_PORT=8450 +REACT_APP_CONTRACTS_VERSION_HASH=latest diff --git a/jenkins/env/contracts.production.jenkins.env b/jenkins/env/contracts.production.jenkins.env new file mode 100755 index 0000000..de1fc9d --- /dev/null +++ b/jenkins/env/contracts.production.jenkins.env @@ -0,0 +1,8 @@ +# Variables needed in runtime +MIGRATE_NETWORK=env +WEB3_HOST=concordia-ganache-production +WEB3_PORT=8545 + +CONTRACTS_PROVIDER_HOST=contracts.concordia.ecentrics.net +CONTRACTS_PROVIDER_PORT=8400 +CONTRACTS_VERSION_TAG=stable diff --git a/jenkins/env/contracts.provider.production.env b/jenkins/env/contracts.provider.production.env new file mode 100755 index 0000000..e2b1437 --- /dev/null +++ b/jenkins/env/contracts.provider.production.env @@ -0,0 +1,8 @@ +VIRTUAL_HOST=contracts.concordia.ecentrics.net +VIRTUAL_PORT=8400 +LETSENCRYPT_HOST=contracts.concordia.ecentrics.net +LETSENCRYPT_EMAIL=ecentricsgr@gmail.com + +CONTRACTS_PROVIDER_PORT=8400 +UPLOAD_CONTRACTS_DIRECTORY=/mnt/concordia/contracts/ +CORS_ALLOWED_ORIGINS="concordia.ecentrics.net;http://concordia.ecentrics.net;https://concordia.ecentrics.net;concordia.ecentrics.net:7777;http://concordia.ecentrics.net:7777;https://concordia.ecentrics.net:7777;127.0.0.1;http://127.0.0.1;127.0.0.1:7777;http://127.0.0.1" diff --git a/jenkins/env/contracts.provider.staging.env b/jenkins/env/contracts.provider.staging.env new file mode 100755 index 0000000..489ded6 --- /dev/null +++ b/jenkins/env/contracts.provider.staging.env @@ -0,0 +1,8 @@ +VIRTUAL_HOST=staging.contracts.concordia.ecentrics.net +VIRTUAL_PORT=8450 +LETSENCRYPT_HOST=staging.contracts.concordia.ecentrics.net +LETSENCRYPT_EMAIL=ecentricsgr@gmail.com + +CONTRACTS_PROVIDER_PORT=8450 +UPLOAD_CONTRACTS_DIRECTORY=/mnt/concordia/contracts/ +#CORS_ALLOWED_ORIGINS="staging.concordia.ecentrics.net;http://staging.concordia.ecentrics.net;https://staging.concordia.ecentrics.net;staging.concordia.ecentrics.net:7000;http://staging.concordia.ecentrics.net:7000;https://staging.concordia.ecentrics.net:7000;172.21.0.4;http://172.21.0.4;127.0.0.1;http://127.0.0.1" diff --git a/jenkins/env/contracts.staging.jenkins.env b/jenkins/env/contracts.staging.jenkins.env new file mode 100755 index 0000000..924f433 --- /dev/null +++ b/jenkins/env/contracts.staging.jenkins.env @@ -0,0 +1,8 @@ +# Variables needed in runtime +MIGRATE_NETWORK=env +WEB3_HOST=concordia-ganache-staging +WEB3_PORT=8555 + +CONTRACTS_PROVIDER_HOST=staging.contracts.concordia.ecentrics.net +CONTRACTS_PROVIDER_PORT=8450 +CONTRACTS_VERSION_TAG=latest diff --git a/jenkins/env/contracts.test.jenkins.env b/jenkins/env/contracts.test.jenkins.env new file mode 100755 index 0000000..8489346 --- /dev/null +++ b/jenkins/env/contracts.test.jenkins.env @@ -0,0 +1,4 @@ +# Variables needed in runtime +MIGRATE_NETWORK=env +WEB3_HOST=concordia-ganache-test +WEB3_PORT=8546 diff --git a/jenkins/env/ganache.production.jenkins.env b/jenkins/env/ganache.production.jenkins.env new file mode 100755 index 0000000..c97e078 --- /dev/null +++ b/jenkins/env/ganache.production.jenkins.env @@ -0,0 +1,10 @@ +VIRTUAL_HOST=ganache.ecentrics.net +VIRTUAL_PORT=8545 +LETSENCRYPT_HOST=ganache.ecentrics.net +LETSENCRYPT_EMAIL=ecentricsgr@gmail.com + +ACCOUNTS_NUMBER=1000 +ACCOUNTS_ETHER=100000 +HOST=0.0.0.0 +PORT=8545 +NETWORK_ID=5778 diff --git a/jenkins/env/ganache.staging.jenkins.env b/jenkins/env/ganache.staging.jenkins.env new file mode 100755 index 0000000..ce93acd --- /dev/null +++ b/jenkins/env/ganache.staging.jenkins.env @@ -0,0 +1,10 @@ +VIRTUAL_HOST=staging.ganache.ecentrics.net +VIRTUAL_PORT=8555 +LETSENCRYPT_HOST=staging.ganache.ecentrics.net +LETSENCRYPT_EMAIL=ecentricsgr@gmail.com + +ACCOUNTS_NUMBER=100 +ACCOUNTS_ETHER=1000 +HOST=0.0.0.0 +PORT=8555 +NETWORK_ID=5778 diff --git a/jenkins/env/ganache.test.jenkins.env b/jenkins/env/ganache.test.jenkins.env new file mode 100755 index 0000000..479da1f --- /dev/null +++ b/jenkins/env/ganache.test.jenkins.env @@ -0,0 +1,6 @@ +ACCOUNTS_NUMBER=5 +ACCOUNTS_ETHER=1 +MNEMONIC="myth like bonus scare over problem client lizard pioneer submit female collect" +HOST=0.0.0.0 +PORT=8546 +NETWORK_ID=5778 diff --git a/jenkins/env/pinner.production.jenkins.env b/jenkins/env/pinner.production.jenkins.env new file mode 100755 index 0000000..5558f2c --- /dev/null +++ b/jenkins/env/pinner.production.jenkins.env @@ -0,0 +1,20 @@ +VIRTUAL_HOST=pinner.concordia.ecentrics.net +VIRTUAL_PORT=4444 +LETSENCRYPT_HOST=pinner.concordia.ecentrics.net +LETSENCRYPT_EMAIL=ecentricsgr@gmail.com + +USE_EXTERNAL_CONTRACTS_PROVIDER=true +ORBIT_DIRECTORY=/mnt/concordia/orbitdb + +CONTRACTS_PROVIDER_HOST=contracts.concordia.ecentrics.net +CONTRACTS_PROVIDER_PORT=8400 +CONTRACTS_VERSION_HASH=stable + +PINNER_API_HOST=127.0.0.1 +PINNER_API_PORT=4444 + +RENDEZVOUS_HOST=rendezvous.ecentrics.net +RENDEZVOUS_PORT=9090 + +WEB3_HOST=ganache.ecentrics.net +WEB3_PORT=8545 diff --git a/jenkins/env/pinner.staging.jenkins.env b/jenkins/env/pinner.staging.jenkins.env new file mode 100755 index 0000000..d4998d2 --- /dev/null +++ b/jenkins/env/pinner.staging.jenkins.env @@ -0,0 +1,20 @@ +VIRTUAL_HOST=staging.pinner.concordia.ecentrics.net +VIRTUAL_PORT=5555 +LETSENCRYPT_HOST=staging.pinner.concordia.ecentrics.net +LETSENCRYPT_EMAIL=ecentricsgr@gmail.com + +USE_EXTERNAL_CONTRACTS_PROVIDER=true +ORBIT_DIRECTORY=/mnt/concordia/orbitdb + +CONTRACTS_PROVIDER_HOST=staging.contracts.concordia.ecentrics.net +CONTRACTS_PROVIDER_PORT=8450 +CONTRACTS_VERSION_HASH=latest + +PINNER_API_HOST=127.0.0.1 +PINNER_API_PORT=5555 + +RENDEZVOUS_HOST=rendezvous.ecentrics.net +RENDEZVOUS_PORT=9090 + +WEB3_HOST=staging.ganache.ecentrics.net +WEB3_PORT=8555 From e2e22d831722b7044a12ef76e5501da4fa39084d Mon Sep 17 00:00:00 2001 From: apostolof Date: Sat, 13 Feb 2021 18:11:52 +0200 Subject: [PATCH 18/18] refactor: add discord hook --- jenkins/Jenkinsfile | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/jenkins/Jenkinsfile b/jenkins/Jenkinsfile index d6aab72..424c05d 100755 --- a/jenkins/Jenkinsfile +++ b/jenkins/Jenkinsfile @@ -27,18 +27,38 @@ def pinnerImage def freshGanacheStagingRunning def freshGanacheProductionRunning +def successResultGif = "https://media.giphy.com/media/o75ajIFH0QnQC3nCeD/giphy.gif" +def failResultGif = "https://media.giphy.com/media/ljtfkyTD3PIUZaKWRi/giphy.gif" +def abortResultGif = "https://media.giphy.com/media/IzXmRTmKd0if6/giphy.gif" + pipeline { agent any post { failure { updateGitlabCommitStatus name: 'build', state: 'failed' + + discordSend footer: "Visit Jenkins for more information", result: currentBuild.currentResult, link: env.BUILD_URL, description: """Jenkins Pipeline Build + Last commit included is [${GIT_COMMIT[0..7]}](https://gitlab.com/ecentrics/concordia/-/commit/$GIT_COMMIT) + Build status: ${currentBuild.currentResult} + """, image: failResultGif, thumbnail: "$CONCORDIA_LOGO_URL", title: JOB_NAME, webhookURL: "${DISCORD_WEBHOOK_URL}" } success { updateGitlabCommitStatus name: 'build', state: 'success' + + discordSend footer: "Visit Jenkins for more information", result: currentBuild.currentResult, link: env.BUILD_URL, description: """Jenkins Pipeline Build + Last commit included is [${GIT_COMMIT[0..7]}](https://gitlab.com/ecentrics/concordia/-/commit/$GIT_COMMIT) + Build status: ${currentBuild.currentResult} + """, image: successResultGif, thumbnail: "$CONCORDIA_LOGO_URL", title: JOB_NAME, webhookURL: "${DISCORD_WEBHOOK_URL}" + } + aborted { + discordSend footer: "Visit Jenkins for more information", result: currentBuild.currentResult, link: env.BUILD_URL, description: """Jenkins Pipeline Build + Last commit included is [${GIT_COMMIT[0..7]}](https://gitlab.com/ecentrics/concordia/-/commit/$GIT_COMMIT) + Build status: ${currentBuild.currentResult} + """, image: abortResultGif, thumbnail: "$CONCORDIA_LOGO_URL", title: JOB_NAME, webhookURL: "${DISCORD_WEBHOOK_URL}" } always { - archiveArtifacts artifacts: "reports/${BUILD_NUMBER}/**/* , build/**/*, ganache/*", fingerprint: true + archiveArtifacts artifacts: "reports/${BUILD_NUMBER}/**/* , build/**/*, ganache/*", fingerprint: true, allowEmptyArchive: true sleep 2 sh 'docker images | grep -E "ecentrics/concordia.+tests" | tr -s \' \' | cut -d \' \' -f 3 | xargs --no-run-if-empty docker rmi -f || true' sh 'docker images | grep -E "ecentrics/concordia.+staging" | tr -s \' \' | cut -d \' \' -f 3 | xargs --no-run-if-empty docker rmi -f || true' @@ -55,6 +75,8 @@ pipeline { } environment { DOCKER_BUILDKIT='1' + DISCORD_WEBHOOK_URL="https://discord.com/api/webhooks/810180975580938290/HYYeK8Nqwt0h8Arx3qPpF-szjgLkPDTqbVVKLkzcmqY7ourTpKJCAc6IuCXHd_cxowuK" + CONCORDIA_LOGO_URL="https://i.postimg.cc/MGvgy9Lp/app-logo-circle.png" } stages { @@ -90,9 +112,7 @@ pipeline { steps { script { try { - echo "${BRANCH_NAME}-${BUILD_NUMBER}" def ganacheTestPort = sh(script: "bash ./jenkins/hash_build_properties.sh ${BRANCH_NAME} ${BUILD_NUMBER} | xargs bash ./jenkins/map_to_thousand.sh", returnStdout: true).trim() - echo "$ganacheTestPort" def ganacheTestImage = docker.build( "ecentrics/concordia-ganache",