From 5ead27b005bdc456edaf0f684e30d1f6a9e3224c Mon Sep 17 00:00:00 2001 From: devingfx Date: Thu, 26 Jul 2018 15:15:04 +0200 Subject: [PATCH] Initial commit --- .gitignore | 2 + bma2kumu.js | 89 ++++++++++++++++++++++++++++++++++ certifications2connections.js | 6 +++ esm.js | 15 ++++++ gentleman.js | 108 ++++++++++++++++++++++++++++++++++++++++++ package.json | 20 ++++++++ person2element.js | 11 +++++ 7 files changed, 251 insertions(+) create mode 100644 .gitignore create mode 100644 bma2kumu.js create mode 100644 certifications2connections.js create mode 100644 esm.js create mode 100644 gentleman.js create mode 100644 package.json create mode 100644 person2element.js diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6bc4134 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +node_modules +.cache diff --git a/bma2kumu.js b/bma2kumu.js new file mode 100644 index 0000000..b71d6dc --- /dev/null +++ b/bma2kumu.js @@ -0,0 +1,89 @@ +import { + delayR // call a function with a random delay +, log // console.log in .map or .then +, concat // concat for reduce: [[1,2],3,[4],[5,6]].reduce(concat,[]) > [1,2,3,4,5,6] +, only // only n element for array.filter +, toCSV // transforms an arry of same objects to CSV string (1st line with columns) +, get // nodejs HTTP.get +, stick // stick together an array of string and arguments (for template literals) +, toJson // parses JSON string +, toFile, // fs.writeFile but in .map or .then (ex: p.then( toFile('toto.txt') ) ) +cache +} from './gentleman' + +// templates +import p2e from './person2element' +import c2c from './certifications2connections' + +// shrotcuts +const ROOT = (ss,...pp)=> `https://g1.duniter.org${stick(ss,...pp)}` +global.BMA = (ss,...pp)=> delayR( get, ROOT`${stick(ss,...pp)}` ).then( toJson ) +delayR.amount = 20000 +await cache(`./.cache`) + +// Get last UD to compute relative balance +const relativity = ( + await BMA`/blockchain/block/${ + ( await BMA`/blockchain/with/ud` ) + .result.blocks.pop() + }` + ).dividend / 100 +// console.log( relativity ) + + +console.log( ROOT`/wot/members` ) +// console.log( await BMA`/wot/members` ) +// console.log( (await BMA`/wot/members`).results.filter(only(10)) ) + + +// Load all members then each certifications +let members = await Promise.all( + ( await BMA`/wot/members` ) + .results + .filter( only(50) ) + // .map( person=> ROOT`/wot/certified-by/${person.pubkey}` ) + // .map( log() ) + .map( person=> BMA`/wot/certified-by/${person.pubkey}` ) +) +// console.log( members ) + +// console.log( 'elems: ', toCSV( await Promise.all(members.map(p2e)) ) ) +// process.exit() + +let kumu = { + elements: await Promise.all( members.map(p2e) ) +, connections: members.map(c2c).reduce(concat,[]) +} + +toFile( `./wot-elem.csv` )( + toCSV( kumu.elements ) +) + +// console.log( 'conns: ', toCSV( members.map(c2c).reduce(concat,[]) ) ) + +toFile( `./wot-conn.csv` )( + toCSV( kumu.connections ) +) + +toFile( `./wot.kumu.json` )( + JSON.stringify( kumu ) +) + +// let csv = `${toCSV( members.map(p2e) )} + + +// ${toCSV( members.map(c2c).reduce(concat,[]) )}` + +// let csv = await load() +// .then( json=> json.results.filter((o,i)=>i<5).map(gentlyLoad) ) +// .then( arr=> Promise.all(arr) ) +// // .then( members=> console.log(members) ) +// .then( members=> ` +// ${toCSV( members.map(p2e) )} + + +// ${toCSV( members.map(c2c).reduce(concat,[]) )} + +// `) + +// console.log('Final CVS:', csv ) diff --git a/certifications2connections.js b/certifications2connections.js new file mode 100644 index 0000000..4897ca3 --- /dev/null +++ b/certifications2connections.js @@ -0,0 +1,6 @@ +export default person=> + person.certifications.map( cert=>({ + From: person.uid + , To: cert.uid + , pubkey: cert.pubkey + }) ) \ No newline at end of file diff --git a/esm.js b/esm.js new file mode 100644 index 0000000..a856d6b --- /dev/null +++ b/esm.js @@ -0,0 +1,15 @@ +const params = process.argv.slice(2) + +const moduleStr = params.shift() +const [ esmodule, exported ] = (moduleStr || '').split(':') + +// console.log(params, esmodule, exported) + +let mod = +require('esm')( module, {await: true} ) +( `./${esmodule || require('./package.json').module}` ) + +let toCall = mod[ exported || 'default' ] +typeof toCall == 'function' + ? console.log( toCall( ...process.argv.slice(2) ) || '' ) + : console.log( toCall ) \ No newline at end of file diff --git a/gentleman.js b/gentleman.js new file mode 100644 index 0000000..4089900 --- /dev/null +++ b/gentleman.js @@ -0,0 +1,108 @@ +import http from 'https' +import { readFile, writeFile, mkdir, stat } from 'fs' +import { resolve } from 'path' + + +const params = process.argv.slice(2) +export const args = params.map( p=> p.split('=') ).reduce( (o,a)=> (o[a[0]]=a[1],o), {} ) + +// shorcuts + +export const exists = path=> new Promise((ok,ko)=> stat( path, (err,stat)=> stat && !err ? ok(true) : ok(false) )) + +// loading + +let cacheDir +const _cachePath = url=> cacheDir + '/' + url.replace( + /^https?:\/\/(.*?)\/(.*?)(\?(.*?))?$/ +, (s, domain, path,q, query)=> + `${domain}/${path}${q ? `/${query.replace(/[=&\\\/%]/g,'-')}` : ``}` +) +export const cache = folder=> exists( cacheDir = folder ).then( being=> !being && mkdir(cacheDir,o=>o) ) + +export const get = (url,enc='utf8')=> new Promise( async(ok,ko)=> + log('GET ')( url ) + && + cacheDir && await exists( _cachePath(url) ) ? + readFile( _cachePath(url), 'utf-8', (err,data)=> ok(data) ) + : + http.get( url, res=> { + if (res.statusCode !== 200) { + ko( new Error('Request Failed.\n' + + `Status Code: ${res.statusCode}`) ) + res.resume() + return + } + + res.setEncoding( enc ) + let rawData = '' + res.on('data', chunk=> rawData += chunk ) + res.on('end', async()=> { + ok(rawData) + // console.log(cacheDir, _cachePath(url)) + + cacheDir + && await ensure( _cachePath(url) ) + && toFile( _cachePath(url) )( rawData ) + }) + }) + .on('error', e=> ko(`Got error: ${e.message}`) ) +) + +export const delayR = (action,...args)=> new Promise( ok=> setTimeout( + ()=> ok( action(...args) ) + , Math.random() * delayR.amount +)) +delayR.amount = 15000 + +export const load = person=> fetch(!person ? `/wot/members` : `/wot/certified-by/${person.pubkey}`).then( res=> res.json() ) + +export const gentlyLoad = person=> delayR( load, person ) + +// misc promise + +export const log = str=> o=> console.log(str, o) || o + +// Array + +export const toCSV = arr=> `${Object.keys(arr[0]).map(quoted)} +${arr.map( item=> Object.keys(item).map(key=> item[key]).map(quoted) ).join('\n')}` + +export const concat = (a,o)=> a.concat ( o ) + +export const only = n=> (o,i)=> n ? i < n : 1 + +// String + +export const stick = (ss,...pp)=> ss.map((s,i)=>s+(pp[i]||'')).join('') + +export const quoted = str=> `"${str}"` + +export const firstCase = str=> str.split(/\s/).map( s=> s[0].toUpperCase() + s.slice(1).toLowerCase() ).join(' ') + +export const noAccent = str=> str.normalize("NFKD").replace( /[\u0300-\u036F]/g, "" ) + +// Object + +export const toJson = data=> JSON.parse(data) + +// export const toCSV = data=> `${Object.keys(data[0]).map(JSON.stringify).join(',')} +// ${data.map(o=> Object.keys(o).map( k=> JSON.stringify(o[k]) ).join(',')).join('\n')}` + +// files + +export const toFile = (path,enc='utf8')=> data=> new Promise( (ok,ko)=> + log('File write:')( path ) + && + writeFile(path, data, 'utf8', err=> err && ko(err) || ok(data) ) +) + +export const ensure = path=> Promise.all( + // log( 'ici') ( resolve( __dirname, path ).split(/[\\/]/).slice( 0, -1 )) && + resolve( __dirname, path ).split(/[\\/]/).slice( 0, -1 ) + .map( (folder,i,path)=> path.filter(only(i+1)).join('/') ) + // .map( log('Folder: ') ) + // .map( async folder=> await exists(folder) ) + // .map( async folder=> !(await exists(folder)) ? 'no: create' : folder ) + .map( async folder=> !(await exists(folder)) ? mkdir(folder,o=>o) : folder ) +) \ No newline at end of file diff --git a/package.json b/package.json new file mode 100644 index 0000000..fbc7e48 --- /dev/null +++ b/package.json @@ -0,0 +1,20 @@ +{ + "name": "woot-kumu", + "version": "1.0.0", + "description": "Duniter wot download and CSV formt for kumu", + "main": "esm", + "module": "bma2kumu", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [ + "duniter", + "wot", + "kumu" + ], + "author": "Di Grégorio Thomas", + "license": "MIT", + "dependencies": { + "esm": "^3.0.72" + } +} diff --git a/person2element.js b/person2element.js new file mode 100644 index 0000000..e417217 --- /dev/null +++ b/person2element.js @@ -0,0 +1,11 @@ + +export default async person=> ({ + Label: person.uid +, Type: 'Person' +, pubkey: person.pubkey +, sigDate: person.sigDate +, isMember: person.isMember +, Image: `https://g1.data.duniter.fr/user/profile/${person.pubkey}/_image/avatar.png` +, balance: ( await BMA`/tx/sources/${person.pubkey}` ) + .sources.reduce( (total, source)=> total + source.amount, 0 ) / 100 +})